Debian Codegraph @

extra/protobuf/protobuf-24.4/src/google/protobuf/compiler/command_line_interface.cc

raw
1 // Protocol Buffers - Google's data interchange format
2 // Copyright 2008 Google Inc. All rights reserved.
3 // https://developers.google.com/protocol-buffers/
4 //
5 // Redistribution and use in source and binary forms, with or without
6 // modification, are permitted provided that the following conditions are
7 // met:
8 //
9 // * Redistributions of source code must retain the above copyright
10 // notice, this list of conditions and the following disclaimer.
11 // * Redistributions in binary form must reproduce the above
12 // copyright notice, this list of conditions and the following disclaimer
13 // in the documentation and/or other materials provided with the
14 // distribution.
15 // * Neither the name of Google Inc. nor the names of its
16 // contributors may be used to endorse or promote products derived from
17 // this software without specific prior written permission.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 // Author: [email protected] (Kenton Varda)
32 // Based on original Protocol Buffers design by
33 // Sanjay Ghemawat, Jeff Dean, and others.
34
35 #include "google/protobuf/compiler/command_line_interface.h"
36
37 #include "absl/algorithm/container.h"
38 #include "absl/container/btree_set.h"
39 #include "absl/container/flat_hash_map.h"
40 #include "absl/types/span.h"
41 #include "google/protobuf/compiler/allowlists/allowlists.h"
42 #include "google/protobuf/descriptor_legacy.h"
43 #include "google/protobuf/descriptor_visitor.h"
44
45 #include "google/protobuf/stubs/platform_macros.h"
46
47 #include <stdio.h>
48 #include <stdlib.h>
49 #include <sys/types.h>
50 #ifdef major
51 #undef major
52 #endif
53 #ifdef minor
54 #undef minor
55 #endif
56 #include <fcntl.h>
57 #include <sys/stat.h>
58 #ifndef _MSC_VER
59 #include <unistd.h>
60 #endif
61 #include <ctype.h>
62 #include <errno.h>
63
64 #include <fstream>
65 #include <iostream>
66
67 #include <limits.h> // For PATH_MAX
68
69 #include <memory>
70 #include <string>
71 #include <utility>
72 #include <vector>
73
74 #if defined(__APPLE__)
75 #include <mach-o/dyld.h>
76 #elif defined(__FreeBSD__)
77 #include <sys/sysctl.h>
78 #endif
79
80 #include "google/protobuf/stubs/common.h"
81 #include "absl/log/absl_check.h"
82 #include "absl/log/absl_log.h"
83 #include "absl/container/flat_hash_set.h"
84 #include "absl/strings/match.h"
85 #include "absl/strings/str_format.h"
86 #include "absl/strings/str_replace.h"
87 #include "absl/strings/str_split.h"
88 #include "absl/strings/string_view.h"
89 #include "absl/strings/substitute.h"
90 #include "google/protobuf/compiler/code_generator.h"
91 #include "google/protobuf/compiler/importer.h"
92 #include "google/protobuf/compiler/plugin.pb.h"
93 #include "google/protobuf/compiler/retention.h"
94 #include "google/protobuf/compiler/subprocess.h"
95 #include "google/protobuf/compiler/zip_writer.h"
96 #include "google/protobuf/descriptor.h"
97 #include "google/protobuf/dynamic_message.h"
98 #include "google/protobuf/io/coded_stream.h"
99 #include "google/protobuf/io/io_win32.h"
100 #include "google/protobuf/io/printer.h"
101 #include "google/protobuf/io/zero_copy_stream_impl.h"
102 #include "google/protobuf/text_format.h"
103
104
105 // Must be included last.
106 #include "google/protobuf/port_def.inc"
107
108 namespace google {
109 namespace protobuf {
110 namespace compiler {
111
112 #ifndef O_BINARY
113 #ifdef _O_BINARY
114 #define O_BINARY _O_BINARY
115 #else
116 #define O_BINARY 0 // If this isn't defined, the platform doesn't need it.
117 #endif
118 #endif
119
120 namespace {
121 #if defined(_WIN32)
122 // DO NOT include <io.h>, instead create functions in io_win32.{h,cc} and import
123 // them like we do below.
124 using google::protobuf::io::win32::access;
125 using google::protobuf::io::win32::close;
126 using google::protobuf::io::win32::mkdir;
127 using google::protobuf::io::win32::open;
128 using google::protobuf::io::win32::setmode;
129 using google::protobuf::io::win32::write;
130 #endif
131
132 static const char* kDefaultDirectDependenciesViolationMsg =
133 "File is imported but not declared in --direct_dependencies: %s";
134
135 // Returns true if the text looks like a Windows-style absolute path, starting
136 // with a drive letter. Example: "C:\foo". TODO(kenton): Share this with
137 // copy in importer.cc?
138 static bool IsWindowsAbsolutePath(const std::string& text) {
139 #if defined(_WIN32) || defined(__CYGWIN__)
140 return text.size() >= 3 && text[1] == ':' && isalpha(text[0]) &&
141 (text[2] == '/' || text[2] == '\\') && text.find_last_of(':') == 1;
142 #else
143 return false;
144 #endif
145 }
146
147 void SetFdToTextMode(int fd) {
148 #ifdef _WIN32
149 if (setmode(fd, _O_TEXT) == -1) {
150 // This should never happen, I think.
151 ABSL_LOG(WARNING) << "setmode(" << fd << ", _O_TEXT): " << strerror(errno);
152 }
153 #endif
154 // (Text and binary are the same on non-Windows platforms.)
155 }
156
157 void SetFdToBinaryMode(int fd) {
158 #ifdef _WIN32
159 if (setmode(fd, _O_BINARY) == -1) {
160 // This should never happen, I think.
161 ABSL_LOG(WARNING) << "setmode(" << fd
162 << ", _O_BINARY): " << strerror(errno);
163 }
164 #endif
165 // (Text and binary are the same on non-Windows platforms.)
166 }
167
168 void AddTrailingSlash(std::string* path) {
169 if (!path->empty() && path->at(path->size() - 1) != '/') {
170 path->push_back('/');
171 }
172 }
173
174 bool VerifyDirectoryExists(const std::string& path) {
175 if (path.empty()) return true;
176
177 if (access(path.c_str(), F_OK) == -1) {
178 std::cerr << path << ": " << strerror(errno) << std::endl;
179 return false;
180 } else {
181 return true;
182 }
183 }
184
185 // Try to create the parent directory of the given file, creating the parent's
186 // parent if necessary, and so on. The full file name is actually
187 // (prefix + filename), but we assume |prefix| already exists and only create
188 // directories listed in |filename|.
189 bool TryCreateParentDirectory(const std::string& prefix,
190 const std::string& filename) {
191 // Recursively create parent directories to the output file.
192 // On Windows, both '/' and '\' are valid path separators.
193 std::vector<std::string> parts =
194 absl::StrSplit(filename, absl::ByAnyChar("/\\"), absl::SkipEmpty());
195 std::string path_so_far = prefix;
196 for (int i = 0; i < parts.size() - 1; i++) {
197 path_so_far += parts[i];
198 if (mkdir(path_so_far.c_str(), 0777) != 0) {
199 if (errno != EEXIST) {
200 std::cerr << filename << ": while trying to create directory "
201 << path_so_far << ": " << strerror(errno) << std::endl;
202 return false;
203 }
204 }
205 path_so_far += '/';
206 }
207
208 return true;
209 }
210
211 // Get the absolute path of this protoc binary.
212 bool GetProtocAbsolutePath(std::string* path) {
213 #ifdef _WIN32
214 char buffer[MAX_PATH];
215 int len = GetModuleFileNameA(nullptr, buffer, MAX_PATH);
216 #elif defined(__APPLE__)
217 char buffer[PATH_MAX];
218 int len = 0;
219
220 char dirtybuffer[PATH_MAX];
221 uint32_t size = sizeof(dirtybuffer);
222 if (_NSGetExecutablePath(dirtybuffer, &size) == 0) {
223 realpath(dirtybuffer, buffer);
224 len = strlen(buffer);
225 }
226 #elif defined(__FreeBSD__)
227 char buffer[PATH_MAX];
228 size_t len = PATH_MAX;
229 int mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1};
230 if (sysctl(mib, 4, &buffer, &len, nullptr, 0) != 0) {
231 len = 0;
232 }
233 #else
234 char buffer[PATH_MAX];
235 int len = readlink("/proc/self/exe", buffer, PATH_MAX);
236 #endif
237 if (len > 0) {
238 path->assign(buffer, len);
239 return true;
240 } else {
241 return false;
242 }
243 }
244
245 // Whether a path is where google/protobuf/descriptor.proto and other well-known
246 // type protos are installed.
247 bool IsInstalledProtoPath(absl::string_view path) {
248 // Checking the descriptor.proto file should be good enough.
249 std::string file_path =
250 absl::StrCat(path, "/google/protobuf/descriptor.proto");
251 return access(file_path.c_str(), F_OK) != -1;
252 }
253
254 // Add the paths where google/protobuf/descriptor.proto and other well-known
255 // type protos are installed.
256 void AddDefaultProtoPaths(
257 std::vector<std::pair<std::string, std::string>>* paths) {
258 // TODO(xiaofeng): The code currently only checks relative paths of where
259 // the protoc binary is installed. We probably should make it handle more
260 // cases than that.
261 std::string path_str;
262 if (!GetProtocAbsolutePath(&path_str)) {
263 return;
264 }
265 absl::string_view path(path_str);
266 // Strip the binary name.
267 size_t pos = path.find_last_of("/\\");
268 if (pos == path.npos || pos == 0) {
269 return;
270 }
271 path = path.substr(0, pos);
272 // Check the binary's directory.
273 if (IsInstalledProtoPath(path)) {
274 paths->emplace_back("", path);
275 return;
276 }
277 // Check if there is an include subdirectory.
278 std::string include_path = absl::StrCat(path, "/include");
279 if (IsInstalledProtoPath(include_path)) {
280 paths->emplace_back("", std::move(include_path));
281 return;
282 }
283 // Check if the upper level directory has an "include" subdirectory.
284 pos = path.find_last_of("/\\");
285 if (pos == std::string::npos || pos == 0) {
286 return;
287 }
288 path = path.substr(0, pos);
289 include_path = absl::StrCat(path, "/include");
290 if (IsInstalledProtoPath(include_path)) {
291 paths->emplace_back("", std::move(include_path));
292 return;
293 }
294 }
295
296 std::string PluginName(absl::string_view plugin_prefix,
297 absl::string_view directive) {
298 // Assuming the directive starts with "--" and ends with "_out" or "_opt",
299 // strip the "--" and "_out/_opt" and add the plugin prefix.
300 return absl::StrCat(plugin_prefix, "gen-",
301 directive.substr(2, directive.size() - 6));
302 }
303
304
305 bool EnforceEditionsSupport(
306 const std::string& codegen_name, uint64_t supported_features,
307 const std::vector<const FileDescriptor*>& parsed_files) {
308 if ((supported_features & CodeGenerator::FEATURE_SUPPORTS_EDITIONS) == 0) {
309 for (const auto fd : parsed_files) {
310 if (FileDescriptorLegacy(fd).syntax() ==
311 FileDescriptorLegacy::SYNTAX_EDITIONS) {
312 std::cerr << fd->name() << ": is an editions file, but code generator "
313 << codegen_name
314 << " hasn't been updated to support editions yet. Please ask "
315 "the owner of this code generator to add support or "
316 "switch back to proto2/proto3."
317 << std::endl;
318 return false;
319 }
320 }
321 }
322 return true;
323 }
324
325 } // namespace
326
327 void CommandLineInterface::GetTransitiveDependencies(
328 const FileDescriptor* file,
329 absl::flat_hash_set<const FileDescriptor*>* already_seen,
330 RepeatedPtrField<FileDescriptorProto>* output,
331 const TransitiveDependencyOptions& options) {
332 if (!already_seen->insert(file).second) {
333 // Already saw this file. Skip.
334 return;
335 }
336
337 // Add all dependencies.
338 for (int i = 0; i < file->dependency_count(); i++) {
339 GetTransitiveDependencies(file->dependency(i), already_seen, output,
340 options);
341 }
342
343 // Add this file.
344 FileDescriptorProto* new_descriptor = output->Add();
345 *new_descriptor =
346 google::protobuf::internal::InternalFeatureHelper::GetGeneratorProto(*file);
347 if (options.include_source_code_info) {
348 file->CopySourceCodeInfoTo(new_descriptor);
349 }
350 if (!options.retain_options) {
351 StripSourceRetentionOptions(*file->pool(), *new_descriptor);
352 }
353 if (options.include_json_name) {
354 file->CopyJsonNameTo(new_descriptor);
355 }
356 }
357
358 // A MultiFileErrorCollector that prints errors to stderr.
359 class CommandLineInterface::ErrorPrinter
360 : public MultiFileErrorCollector,
361 public io::ErrorCollector,
362 public DescriptorPool::ErrorCollector {
363 public:
364 ErrorPrinter(ErrorFormat format, DiskSourceTree* tree = nullptr)
365 : format_(format),
366 tree_(tree),
367 found_errors_(false),
368 found_warnings_(false) {}
369 ~ErrorPrinter() override {}
370
371 // implements MultiFileErrorCollector ------------------------------
372 void RecordError(absl::string_view filename, int line, int column,
373 absl::string_view message) override {
374 found_errors_ = true;
375 AddErrorOrWarning(filename, line, column, message, "error", std::cerr);
376 }
377
378 void RecordWarning(absl::string_view filename, int line, int column,
379 absl::string_view message) override {
380 found_warnings_ = true;
381 AddErrorOrWarning(filename, line, column, message, "warning", std::clog);
382 }
383
384 // implements io::ErrorCollector -----------------------------------
385 void RecordError(int line, int column, absl::string_view message) override {
386 RecordError("input", line, column, message);
387 }
388
389 void RecordWarning(int line, int column, absl::string_view message) override {
390 AddErrorOrWarning("input", line, column, message, "warning", std::clog);
391 }
392
393 // implements DescriptorPool::ErrorCollector-------------------------
394 void RecordError(absl::string_view filename, absl::string_view element_name,
395 const Message* descriptor, ErrorLocation location,
396 absl::string_view message) override {
397 AddErrorOrWarning(filename, -1, -1, message, "error", std::cerr);
398 }
399
400 void RecordWarning(absl::string_view filename, absl::string_view element_name,
401 const Message* descriptor, ErrorLocation location,
402 absl::string_view message) override {
403 AddErrorOrWarning(filename, -1, -1, message, "warning", std::clog);
404 }
405
406 bool FoundErrors() const { return found_errors_; }
407
408 bool FoundWarnings() const { return found_warnings_; }
409
410 private:
411 void AddErrorOrWarning(absl::string_view filename, int line, int column,
412 absl::string_view message, absl::string_view type,
413 std::ostream& out) {
414 std::string dfile;
415 if (
416 #ifndef PROTOBUF_OPENSOURCE
417 // Print full path when running under MSVS
418 format_ == CommandLineInterface::ERROR_FORMAT_MSVS &&
419 #endif // !PROTOBUF_OPENSOURCE
420 tree_ != nullptr && tree_->VirtualFileToDiskFile(filename, &dfile)) {
421 out << dfile;
422 } else {
423 out << filename;
424 }
425
426 // Users typically expect 1-based line/column numbers, so we add 1
427 // to each here.
428 if (line != -1) {
429 // Allow for both GCC- and Visual-Studio-compatible output.
430 switch (format_) {
431 case CommandLineInterface::ERROR_FORMAT_GCC:
432 out << ":" << (line + 1) << ":" << (column + 1);
433 break;
434 case CommandLineInterface::ERROR_FORMAT_MSVS:
435 out << "(" << (line + 1) << ") : " << type
436 << " in column=" << (column + 1);
437 break;
438 }
439 }
440
441 if (type == "warning") {
442 out << ": warning: " << message << std::endl;
443 } else {
444 out << ": " << message << std::endl;
445 }
446 }
447
448 const ErrorFormat format_;
449 DiskSourceTree* tree_;
450 bool found_errors_;
451 bool found_warnings_;
452 };
453
454 // -------------------------------------------------------------------
455
456 // A GeneratorContext implementation that buffers files in memory, then dumps
457 // them all to disk on demand.
458 class CommandLineInterface::GeneratorContextImpl : public GeneratorContext {
459 public:
460 GeneratorContextImpl(const std::vector<const FileDescriptor*>& parsed_files);
461
462 // Write all files in the directory to disk at the given output location,
463 // which must end in a '/'.
464 bool WriteAllToDisk(const std::string& prefix);
465
466 // Write the contents of this directory to a ZIP-format archive with the
467 // given name.
468 bool WriteAllToZip(const std::string& filename);
469
470 // Add a boilerplate META-INF/MANIFEST.MF file as required by the Java JAR
471 // format, unless one has already been written.
472 void AddJarManifest();
473
474 // Get name of all output files.
475 void GetOutputFilenames(std::vector<std::string>* output_filenames);
476 // implements GeneratorContext --------------------------------------
477 io::ZeroCopyOutputStream* Open(const std::string& filename) override;
478 io::ZeroCopyOutputStream* OpenForAppend(const std::string& filename) override;
479 io::ZeroCopyOutputStream* OpenForInsert(
480 const std::string& filename, const std::string& insertion_point) override;
481 io::ZeroCopyOutputStream* OpenForInsertWithGeneratedCodeInfo(
482 const std::string& filename, const std::string& insertion_point,
483 const google::protobuf::GeneratedCodeInfo& info) override;
484 void ListParsedFiles(std::vector<const FileDescriptor*>* output) override {
485 *output = parsed_files_;
486 }
487
488 private:
489 friend class MemoryOutputStream;
490
491 // The files_ field maps from path keys to file content values. It's a map
492 // instead of an unordered_map so that files are written in order (good when
493 // writing zips).
494 absl::btree_map<std::string, std::string> files_;
495 const std::vector<const FileDescriptor*>& parsed_files_;
496 bool had_error_;
497 };
498
499 class CommandLineInterface::MemoryOutputStream
500 : public io::ZeroCopyOutputStream {
501 public:
502 MemoryOutputStream(GeneratorContextImpl* directory,
503 const std::string& filename, bool append_mode);
504 MemoryOutputStream(GeneratorContextImpl* directory,
505 const std::string& filename,
506 const std::string& insertion_point);
507 MemoryOutputStream(GeneratorContextImpl* directory,
508 const std::string& filename,
509 const std::string& insertion_point,
510 const google::protobuf::GeneratedCodeInfo& info);
511 ~MemoryOutputStream() override;
512
513 // implements ZeroCopyOutputStream ---------------------------------
514 bool Next(void** data, int* size) override {
515 return inner_->Next(data, size);
516 }
517 void BackUp(int count) override { inner_->BackUp(count); }
518 int64_t ByteCount() const override { return inner_->ByteCount(); }
519
520 private:
521 // Checks to see if "filename_.pb.meta" exists in directory_; if so, fixes the
522 // offsets in that GeneratedCodeInfo record to reflect bytes inserted in
523 // filename_ at original offset insertion_offset with length insertion_length.
524 // Also adds in the data from info_to_insert_ with updated offsets governed by
525 // insertion_offset and indent_length. We assume that insertions will not
526 // occur within any given annotated span of text. insertion_content must end
527 // with an endline.
528 void UpdateMetadata(const std::string& insertion_content,
529 size_t insertion_offset, size_t insertion_length,
530 size_t indent_length);
531
532 // Inserts info_to_insert_ into target_info, assuming that the relevant
533 // insertion was made at insertion_offset in file_content with the given
534 // indent_length. insertion_content must end with an endline.
535 void InsertShiftedInfo(const std::string& insertion_content,
536 size_t insertion_offset, size_t indent_length,
537 google::protobuf::GeneratedCodeInfo& target_info);
538
539 // Where to insert the string when it's done.
540 GeneratorContextImpl* directory_;
541 std::string filename_;
542 std::string insertion_point_;
543
544 // The string we're building.
545 std::string data_;
546
547 // Whether we should append the output stream to the existing file.
548 bool append_mode_;
549
550 // StringOutputStream writing to data_.
551 std::unique_ptr<io::StringOutputStream> inner_;
552
553 // The GeneratedCodeInfo to insert at the insertion point.
554 google::protobuf::GeneratedCodeInfo info_to_insert_;
555 };
556
557 // -------------------------------------------------------------------
558
559 CommandLineInterface::GeneratorContextImpl::GeneratorContextImpl(
560 const std::vector<const FileDescriptor*>& parsed_files)
561 : parsed_files_(parsed_files), had_error_(false) {}
562
563 bool CommandLineInterface::GeneratorContextImpl::WriteAllToDisk(
564 const std::string& prefix) {
565 if (had_error_) {
566 return false;
567 }
568
569 if (!VerifyDirectoryExists(prefix)) {
570 return false;
571 }
572
573 for (const auto& pair : files_) {
574 const std::string& relative_filename = pair.first;
575 const char* data = pair.second.data();
576 int size = pair.second.size();
577
578 if (!TryCreateParentDirectory(prefix, relative_filename)) {
579 return false;
580 }
581 std::string filename = prefix + relative_filename;
582
583 // Create the output file.
584 int file_descriptor;
585 do {
586 file_descriptor =
587 open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666);
588 } while (file_descriptor < 0 && errno == EINTR);
589
590 if (file_descriptor < 0) {
591 int error = errno;
592 std::cerr << filename << ": " << strerror(error);
593 return false;
594 }
595
596 // Write the file.
597 while (size > 0) {
598 int write_result;
599 do {
600 write_result = write(file_descriptor, data, size);
601 } while (write_result < 0 && errno == EINTR);
602
603 if (write_result <= 0) {
604 // Write error.
605
606 // FIXME(kenton): According to the man page, if write() returns zero,
607 // there was no error; write() simply did not write anything. It's
608 // unclear under what circumstances this might happen, but presumably
609 // errno won't be set in this case. I am confused as to how such an
610 // event should be handled. For now I'm treating it as an error,
611 // since retrying seems like it could lead to an infinite loop. I
612 // suspect this never actually happens anyway.
613
614 if (write_result < 0) {
615 int error = errno;
616 std::cerr << filename << ": write: " << strerror(error);
617 } else {
618 std::cerr << filename << ": write() returned zero?" << std::endl;
619 }
620 return false;
621 }
622
623 data += write_result;
624 size -= write_result;
625 }
626
627 if (close(file_descriptor) != 0) {
628 int error = errno;
629 std::cerr << filename << ": close: " << strerror(error);
630 return false;
631 }
632 }
633
634 return true;
635 }
636
637 bool CommandLineInterface::GeneratorContextImpl::WriteAllToZip(
638 const std::string& filename) {
639 if (had_error_) {
640 return false;
641 }
642
643 // Create the output file.
644 int file_descriptor;
645 do {
646 file_descriptor =
647 open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666);
648 } while (file_descriptor < 0 && errno == EINTR);
649
650 if (file_descriptor < 0) {
651 int error = errno;
652 std::cerr << filename << ": " << strerror(error);
653 return false;
654 }
655
656 // Create the ZipWriter
657 io::FileOutputStream stream(file_descriptor);
658 ZipWriter zip_writer(&stream);
659
660 for (const auto& pair : files_) {
661 zip_writer.Write(pair.first, pair.second);
662 }
663
664 zip_writer.WriteDirectory();
665
666 if (stream.GetErrno() != 0) {
667 std::cerr << filename << ": " << strerror(stream.GetErrno()) << std::endl;
668 return false;
669 }
670
671 if (!stream.Close()) {
672 std::cerr << filename << ": " << strerror(stream.GetErrno()) << std::endl;
673 return false;
674 }
675
676 return true;
677 }
678
679 void CommandLineInterface::GeneratorContextImpl::AddJarManifest() {
680 auto pair = files_.insert({"META-INF/MANIFEST.MF", ""});
681 if (pair.second) {
682 pair.first->second =
683 "Manifest-Version: 1.0\n"
684 "Created-By: 1.6.0 (protoc)\n"
685 "\n";
686 }
687 }
688
689 void CommandLineInterface::GeneratorContextImpl::GetOutputFilenames(
690 std::vector<std::string>* output_filenames) {
691 for (const auto& pair : files_) {
692 output_filenames->push_back(pair.first);
693 }
694 }
695
696 io::ZeroCopyOutputStream* CommandLineInterface::GeneratorContextImpl::Open(
697 const std::string& filename) {
698 return new MemoryOutputStream(this, filename, false);
699 }
700
701 io::ZeroCopyOutputStream*
702 CommandLineInterface::GeneratorContextImpl::OpenForAppend(
703 const std::string& filename) {
704 return new MemoryOutputStream(this, filename, true);
705 }
706
707 io::ZeroCopyOutputStream*
708 CommandLineInterface::GeneratorContextImpl::OpenForInsert(
709 const std::string& filename, const std::string& insertion_point) {
710 return new MemoryOutputStream(this, filename, insertion_point);
711 }
712
713 io::ZeroCopyOutputStream*
714 CommandLineInterface::GeneratorContextImpl::OpenForInsertWithGeneratedCodeInfo(
715 const std::string& filename, const std::string& insertion_point,
716 const google::protobuf::GeneratedCodeInfo& info) {
717 return new MemoryOutputStream(this, filename, insertion_point, info);
718 }
719
720 // -------------------------------------------------------------------
721
722 CommandLineInterface::MemoryOutputStream::MemoryOutputStream(
723 GeneratorContextImpl* directory, const std::string& filename,
724 bool append_mode)
725 : directory_(directory),
726 filename_(filename),
727 append_mode_(append_mode),
728 inner_(new io::StringOutputStream(&data_)) {}
729
730 CommandLineInterface::MemoryOutputStream::MemoryOutputStream(
731 GeneratorContextImpl* directory, const std::string& filename,
732 const std::string& insertion_point)
733 : directory_(directory),
734 filename_(filename),
735 insertion_point_(insertion_point),
736 inner_(new io::StringOutputStream(&data_)) {}
737
738 CommandLineInterface::MemoryOutputStream::MemoryOutputStream(
739 GeneratorContextImpl* directory, const std::string& filename,
740 const std::string& insertion_point, const google::protobuf::GeneratedCodeInfo& info)
741 : directory_(directory),
742 filename_(filename),
743 insertion_point_(insertion_point),
744 inner_(new io::StringOutputStream(&data_)),
745 info_to_insert_(info) {}
746
747 void CommandLineInterface::MemoryOutputStream::InsertShiftedInfo(
748 const std::string& insertion_content, size_t insertion_offset,
749 size_t indent_length, google::protobuf::GeneratedCodeInfo& target_info) {
750 // Keep track of how much extra data was added for indents before the
751 // current annotation being inserted. `pos` and `source_annotation.begin()`
752 // are offsets in `insertion_content`. `insertion_offset` is updated so that
753 // it can be added to an annotation's `begin` field to reflect that
754 // annotation's updated location after `insertion_content` was inserted into
755 // the target file.
756 size_t pos = 0;
757 insertion_offset += indent_length;
758 for (const auto& source_annotation : info_to_insert_.annotation()) {
759 GeneratedCodeInfo::Annotation* annotation = target_info.add_annotation();
760 int inner_indent = 0;
761 // insertion_content is guaranteed to end in an endline. This last endline
762 // has no effect on indentation.
763 for (; pos < source_annotation.end() && pos < insertion_content.size() - 1;
764 ++pos) {
765 if (insertion_content[pos] == '\n') {
766 if (pos >= source_annotation.begin()) {
767 // The beginning of the annotation is at insertion_offset, but the end
768 // can still move further in the target file.
769 inner_indent += indent_length;
770 } else {
771 insertion_offset += indent_length;
772 }
773 }
774 }
775 *annotation = source_annotation;
776 annotation->set_begin(annotation->begin() + insertion_offset);
777 insertion_offset += inner_indent;
778 annotation->set_end(annotation->end() + insertion_offset);
779 }
780 }
781
782 void CommandLineInterface::MemoryOutputStream::UpdateMetadata(
783 const std::string& insertion_content, size_t insertion_offset,
784 size_t insertion_length, size_t indent_length) {
785 auto it = directory_->files_.find(absl::StrCat(filename_, ".pb.meta"));
786 if (it == directory_->files_.end() && info_to_insert_.annotation().empty()) {
787 // No metadata was recorded for this file.
788 return;
789 }
790 GeneratedCodeInfo metadata;
791 bool is_text_format = false;
792 std::string* encoded_data = nullptr;
793 if (it != directory_->files_.end()) {
794 encoded_data = &it->second;
795 // Try to decode a GeneratedCodeInfo proto from the .pb.meta file. It may be
796 // in wire or text format. Keep the same format when the data is written out
797 // later.
798 if (!metadata.ParseFromString(*encoded_data)) {
799 if (!TextFormat::ParseFromString(*encoded_data, &metadata)) {
800 // The metadata is invalid.
801 std::cerr
802 << filename_
803 << ".pb.meta: Could not parse metadata as wire or text format."
804 << std::endl;
805 return;
806 }
807 // Generators that use the public plugin interface emit text-format
808 // metadata (because in the public plugin protocol, file content must be
809 // UTF8-encoded strings).
810 is_text_format = true;
811 }
812 } else {
813 // Create a new file to store the new metadata in info_to_insert_.
814 encoded_data =
815 &directory_->files_.try_emplace(absl::StrCat(filename_, ".pb.meta"), "")
816 .first->second;
817 }
818 GeneratedCodeInfo new_metadata;
819 bool crossed_offset = false;
820 size_t to_add = 0;
821 for (const auto& source_annotation : metadata.annotation()) {
822 // The first time an annotation at or after the insertion point is found,
823 // insert the new metadata from info_to_insert_. Shift all annotations
824 // after the new metadata by the length of the text that was inserted
825 // (including any additional indent length).
826 if (source_annotation.begin() >= insertion_offset && !crossed_offset) {
827 crossed_offset = true;
828 InsertShiftedInfo(insertion_content, insertion_offset, indent_length,
829 new_metadata);
830 to_add += insertion_length;
831 }
832 GeneratedCodeInfo::Annotation* annotation = new_metadata.add_annotation();
833 *annotation = source_annotation;
834 annotation->set_begin(annotation->begin() + to_add);
835 annotation->set_end(annotation->end() + to_add);
836 }
837 // If there were never any annotations at or after the insertion point,
838 // make sure to still insert the new metadata from info_to_insert_.
839 if (!crossed_offset) {
840 InsertShiftedInfo(insertion_content, insertion_offset, indent_length,
841 new_metadata);
842 }
843 if (is_text_format) {
844 TextFormat::PrintToString(new_metadata, encoded_data);
845 } else {
846 new_metadata.SerializeToString(encoded_data);
847 }
848 }
849
850 CommandLineInterface::MemoryOutputStream::~MemoryOutputStream() {
851 // Make sure all data has been written.
852 inner_.reset();
853
854 // Insert into the directory.
855 auto pair = directory_->files_.insert({filename_, ""});
856 auto it = pair.first;
857 bool already_present = !pair.second;
858
859 if (insertion_point_.empty()) {
860 // This was just a regular Open().
861 if (already_present) {
862 if (append_mode_) {
863 it->second.append(data_);
864 } else {
865 std::cerr << filename_ << ": Tried to write the same file twice."
866 << std::endl;
867 directory_->had_error_ = true;
868 }
869 return;
870 }
871
872 it->second.swap(data_);
873 return;
874 }
875 // This was an OpenForInsert().
876
877 // If the data doesn't end with a clean line break, add one.
878 if (!data_.empty() && data_[data_.size() - 1] != '\n') {
879 data_.push_back('\n');
880 }
881
882 // Find the file we are going to insert into.
883 if (!already_present) {
884 std::cerr << filename_ << ": Tried to insert into file that doesn't exist."
885 << std::endl;
886 directory_->had_error_ = true;
887 return;
888 }
889 std::string* target = &it->second;
890
891 // Find the insertion point.
892 std::string magic_string =
893 absl::Substitute("@@protoc_insertion_point($0)", insertion_point_);
894 std::string::size_type pos = target->find(magic_string);
895
896 if (pos == std::string::npos) {
897 std::cerr << filename_ << ": insertion point \"" << insertion_point_
898 << "\" not found." << std::endl;
899 directory_->had_error_ = true;
900 return;
901 }
902
903 if ((pos > 3) && (target->substr(pos - 3, 2) == "/*")) {
904 // Support for inline "/* @@protoc_insertion_point() */"
905 pos = pos - 3;
906 } else {
907 // Seek backwards to the beginning of the line, which is where we will
908 // insert the data. Note that this has the effect of pushing the
909 // insertion point down, so the data is inserted before it. This is
910 // intentional because it means that multiple insertions at the same point
911 // will end up in the expected order in the final output.
912 pos = target->find_last_of('\n', pos);
913 if (pos == std::string::npos) {
914 // Insertion point is on the first line.
915 pos = 0;
916 } else {
917 // Advance to character after '\n'.
918 ++pos;
919 }
920 }
921
922 // Extract indent.
923 std::string indent_(*target, pos,
924 target->find_first_not_of(" \t", pos) - pos);
925
926 if (indent_.empty()) {
927 // No indent. This makes things easier.
928 target->insert(pos, data_);
929 UpdateMetadata(data_, pos, data_.size(), 0);
930 return;
931 }
932 // Calculate how much space we need.
933 int indent_size = 0;
934 for (int i = 0; i < data_.size(); i++) {
935 if (data_[i] == '\n') indent_size += indent_.size();
936 }
937
938 // Make a hole for it.
939 target->insert(pos, data_.size() + indent_size, '\0');
940
941 // Now copy in the data.
942 std::string::size_type data_pos = 0;
943 char* target_ptr = &(*target)[pos];
944 while (data_pos < data_.size()) {
945 // Copy indent.
946 memcpy(target_ptr, indent_.data(), indent_.size());
947 target_ptr += indent_.size();
948
949 // Copy line from data_.
950 // We already guaranteed that data_ ends with a newline (above), so this
951 // search can't fail.
952 std::string::size_type line_length =
953 data_.find_first_of('\n', data_pos) + 1 - data_pos;
954 memcpy(target_ptr, data_.data() + data_pos, line_length);
955 target_ptr += line_length;
956 data_pos += line_length;
957 }
958
959 ABSL_CHECK_EQ(target_ptr, &(*target)[pos] + data_.size() + indent_size);
960
961 UpdateMetadata(data_, pos, data_.size() + indent_size, indent_.size());
962 }
963
964 // ===================================================================
965
966 #if defined(_WIN32) && !defined(__CYGWIN__)
967 const char* const CommandLineInterface::kPathSeparator = ";";
968 #else
969 const char* const CommandLineInterface::kPathSeparator = ":";
970 #endif
971
972 CommandLineInterface::CommandLineInterface()
973 : direct_dependencies_violation_msg_(
974 kDefaultDirectDependenciesViolationMsg) {}
975
976 CommandLineInterface::~CommandLineInterface() {}
977
978 void CommandLineInterface::RegisterGenerator(const std::string& flag_name,
979 CodeGenerator* generator,
980 const std::string& help_text) {
981 GeneratorInfo info;
982 info.flag_name = flag_name;
983 info.generator = generator;
984 info.help_text = help_text;
985 generators_by_flag_name_[flag_name] = info;
986 }
987
988 void CommandLineInterface::RegisterGenerator(
989 const std::string& flag_name, const std::string& option_flag_name,
990 CodeGenerator* generator, const std::string& help_text) {
991 GeneratorInfo info;
992 info.flag_name = flag_name;
993 info.option_flag_name = option_flag_name;
994 info.generator = generator;
995 info.help_text = help_text;
996 generators_by_flag_name_[flag_name] = info;
997 generators_by_option_name_[option_flag_name] = info;
998 }
999
1000 void CommandLineInterface::AllowPlugins(const std::string& exe_name_prefix) {
1001 plugin_prefix_ = exe_name_prefix;
1002 }
1003
1004 namespace {
1005
1006 bool ContainsProto3Optional(const Descriptor* desc) {
1007 for (int i = 0; i < desc->field_count(); i++) {
1008 if (FieldDescriptorLegacy(desc->field(i)).has_optional_keyword()) {
1009 return true;
1010 }
1011 }
1012 for (int i = 0; i < desc->nested_type_count(); i++) {
1013 if (ContainsProto3Optional(desc->nested_type(i))) {
1014 return true;
1015 }
1016 }
1017 return false;
1018 }
1019
1020 bool ContainsProto3Optional(const FileDescriptor* file) {
1021 if (FileDescriptorLegacy(file).syntax() ==
1022 FileDescriptorLegacy::Syntax::SYNTAX_PROTO3) {
1023 for (int i = 0; i < file->message_type_count(); i++) {
1024 if (ContainsProto3Optional(file->message_type(i))) {
1025 return true;
1026 }
1027 }
1028 }
1029 return false;
1030 }
1031
1032 bool HasReservedFieldNumber(const FieldDescriptor* field) {
1033 if (field->number() >= FieldDescriptor::kFirstReservedNumber &&
1034 field->number() <= FieldDescriptor::kLastReservedNumber) {
1035 return true;
1036 }
1037 return false;
1038 }
1039
1040 } // namespace
1041
1042 namespace {
1043 std::unique_ptr<SimpleDescriptorDatabase>
1044 PopulateSingleSimpleDescriptorDatabase(const std::string& descriptor_set_name);
1045
1046 // Indicates whether the field is compatible with the given target type.
1047 bool IsFieldCompatible(const FieldDescriptor& field,
1048 FieldOptions::OptionTargetType target_type) {
1049 const RepeatedField<int>& allowed_targets = field.options().targets();
1050 return allowed_targets.empty() ||
1051 absl::c_linear_search(allowed_targets, target_type);
1052 }
1053
1054 // Converts the OptionTargetType enum to a string suitable for use in error
1055 // messages.
1056 absl::string_view TargetTypeString(FieldOptions::OptionTargetType target_type) {
1057 switch (target_type) {
1058 case FieldOptions::TARGET_TYPE_FILE:
1059 return "file";
1060 case FieldOptions::TARGET_TYPE_EXTENSION_RANGE:
1061 return "extension range";
1062 case FieldOptions::TARGET_TYPE_MESSAGE:
1063 return "message";
1064 case FieldOptions::TARGET_TYPE_FIELD:
1065 return "field";
1066 case FieldOptions::TARGET_TYPE_ONEOF:
1067 return "oneof";
1068 case FieldOptions::TARGET_TYPE_ENUM:
1069 return "enum";
1070 case FieldOptions::TARGET_TYPE_ENUM_ENTRY:
1071 return "enum entry";
1072 case FieldOptions::TARGET_TYPE_SERVICE:
1073 return "service";
1074 case FieldOptions::TARGET_TYPE_METHOD:
1075 return "method";
1076 default:
1077 return "unknown";
1078 }
1079 }
1080
1081 // Recursively validates that the options message (or subpiece of an options
1082 // message) is compatible with the given target type.
1083 bool ValidateTargetConstraintsRecursive(
1084 const Message& m, DescriptorPool::ErrorCollector& error_collector,
1085 absl::string_view file_name, FieldOptions::OptionTargetType target_type) {
1086 std::vector<const FieldDescriptor*> fields;
1087 const Reflection* reflection = m.GetReflection();
1088 reflection->ListFields(m, &fields);
1089 bool success = true;
1090 for (const auto* field : fields) {
1091 if (!IsFieldCompatible(*field, target_type)) {
1092 success = false;
1093 error_collector.RecordError(
1094 file_name, "", nullptr, DescriptorPool::ErrorCollector::OPTION_NAME,
1095 absl::StrCat("Option ", field->full_name(),
1096 " cannot be set on an entity of type `",
1097 TargetTypeString(target_type), "`."));
1098 }
1099 if (field->type() == FieldDescriptor::TYPE_MESSAGE) {
1100 if (field->is_repeated()) {
1101 int field_size = reflection->FieldSize(m, field);
1102 for (int i = 0; i < field_size; ++i) {
1103 if (!ValidateTargetConstraintsRecursive(
1104 reflection->GetRepeatedMessage(m, field, i), error_collector,
1105 file_name, target_type)) {
1106 success = false;
1107 }
1108 }
1109 } else if (!ValidateTargetConstraintsRecursive(
1110 reflection->GetMessage(m, field), error_collector,
1111 file_name, target_type)) {
1112 success = false;
1113 }
1114 }
1115 }
1116 return success;
1117 }
1118
1119 // Validates that the options message is correct with respect to target
1120 // constraints, returning true if successful. This function converts the
1121 // options message to a DynamicMessage so that we have visibility into custom
1122 // options. We take the element name as a FunctionRef so that we do not have to
1123 // pay the cost of constructing it unless there is an error.
1124 bool ValidateTargetConstraints(const Message& options,
1125 const DescriptorPool& pool,
1126 DescriptorPool::ErrorCollector& error_collector,
1127 absl::string_view file_name,
1128 FieldOptions::OptionTargetType target_type) {
1129 const Descriptor* descriptor =
1130 pool.FindMessageTypeByName(options.GetTypeName());
1131 if (descriptor == nullptr) {
1132 // We were unable to find the options message in the descriptor pool. This
1133 // implies that the proto files we are working with do not depend on
1134 // descriptor.proto, in which case there are no custom options to worry
1135 // about. We can therefore skip the use of DynamicMessage.
1136 return ValidateTargetConstraintsRecursive(options, error_collector,
1137 file_name, target_type);
1138 } else {
1139 DynamicMessageFactory factory;
1140 std::unique_ptr<Message> dynamic_message(
1141 factory.GetPrototype(descriptor)->New());
1142 std::string serialized;
1143 ABSL_CHECK(options.SerializeToString(&serialized));
1144 ABSL_CHECK(dynamic_message->ParseFromString(serialized));
1145 return ValidateTargetConstraintsRecursive(*dynamic_message, error_collector,
1146 file_name, target_type);
1147 }
1148 }
1149
1150 // The overloaded GetTargetType() functions below allow us to map from a
1151 // descriptor type to the associated OptionTargetType enum.
1152 FieldOptions::OptionTargetType GetTargetType(const FileDescriptor*) {
1153 return FieldOptions::TARGET_TYPE_FILE;
1154 }
1155
1156 FieldOptions::OptionTargetType GetTargetType(
1157 const Descriptor::ExtensionRange*) {
1158 return FieldOptions::TARGET_TYPE_EXTENSION_RANGE;
1159 }
1160
1161 FieldOptions::OptionTargetType GetTargetType(const Descriptor*) {
1162 return FieldOptions::TARGET_TYPE_MESSAGE;
1163 }
1164
1165 FieldOptions::OptionTargetType GetTargetType(const FieldDescriptor*) {
1166 return FieldOptions::TARGET_TYPE_FIELD;
1167 }
1168
1169 FieldOptions::OptionTargetType GetTargetType(const OneofDescriptor*) {
1170 return FieldOptions::TARGET_TYPE_ONEOF;
1171 }
1172
1173 FieldOptions::OptionTargetType GetTargetType(const EnumDescriptor*) {
1174 return FieldOptions::TARGET_TYPE_ENUM;
1175 }
1176
1177 FieldOptions::OptionTargetType GetTargetType(const EnumValueDescriptor*) {
1178 return FieldOptions::TARGET_TYPE_ENUM_ENTRY;
1179 }
1180
1181 FieldOptions::OptionTargetType GetTargetType(const ServiceDescriptor*) {
1182 return FieldOptions::TARGET_TYPE_SERVICE;
1183 }
1184
1185 FieldOptions::OptionTargetType GetTargetType(const MethodDescriptor*) {
1186 return FieldOptions::TARGET_TYPE_METHOD;
1187 }
1188 } // namespace
1189
1190 int CommandLineInterface::Run(int argc, const char* const argv[]) {
1191 Clear();
1192
1193 switch (ParseArguments(argc, argv)) {
1194 case PARSE_ARGUMENT_DONE_AND_EXIT:
1195 return 0;
1196 case PARSE_ARGUMENT_FAIL:
1197 return 1;
1198 case PARSE_ARGUMENT_DONE_AND_CONTINUE:
1199 break;
1200 }
1201
1202 std::vector<const FileDescriptor*> parsed_files;
1203 std::unique_ptr<DiskSourceTree> disk_source_tree;
1204 std::unique_ptr<ErrorPrinter> error_collector;
1205 std::unique_ptr<DescriptorPool> descriptor_pool;
1206
1207 // The SimpleDescriptorDatabases here are the constituents of the
1208 // MergedDescriptorDatabase descriptor_set_in_database, so this vector is for
1209 // managing their lifetimes. Its scope should match descriptor_set_in_database
1210 std::vector<std::unique_ptr<SimpleDescriptorDatabase>>
1211 databases_per_descriptor_set;
1212 std::unique_ptr<MergedDescriptorDatabase> descriptor_set_in_database;
1213
1214 std::unique_ptr<SourceTreeDescriptorDatabase> source_tree_database;
1215
1216 // Any --descriptor_set_in FileDescriptorSet objects will be used as a
1217 // fallback to input_files on command line, so create that db first.
1218 if (!descriptor_set_in_names_.empty()) {
1219 for (const std::string& name : descriptor_set_in_names_) {
1220 std::unique_ptr<SimpleDescriptorDatabase> database_for_descriptor_set =
1221 PopulateSingleSimpleDescriptorDatabase(name);
1222 if (!database_for_descriptor_set) {
1223 return EXIT_FAILURE;
1224 }
1225 databases_per_descriptor_set.push_back(
1226 std::move(database_for_descriptor_set));
1227 }
1228
1229 std::vector<DescriptorDatabase*> raw_databases_per_descriptor_set;
1230 raw_databases_per_descriptor_set.reserve(
1231 databases_per_descriptor_set.size());
1232 for (const std::unique_ptr<SimpleDescriptorDatabase>& db :
1233 databases_per_descriptor_set) {
1234 raw_databases_per_descriptor_set.push_back(db.get());
1235 }
1236 descriptor_set_in_database.reset(
1237 new MergedDescriptorDatabase(raw_databases_per_descriptor_set));
1238 }
1239
1240 if (proto_path_.empty()) {
1241 // If there are no --proto_path flags, then just look in the specified
1242 // --descriptor_set_in files. But first, verify that the input files are
1243 // there.
1244 if (!VerifyInputFilesInDescriptors(descriptor_set_in_database.get())) {
1245 return 1;
1246 }
1247
1248 error_collector.reset(new ErrorPrinter(error_format_));
1249 descriptor_pool.reset(new DescriptorPool(descriptor_set_in_database.get(),
1250 error_collector.get()));
1251 } else {
1252 disk_source_tree.reset(new DiskSourceTree());
1253 if (!InitializeDiskSourceTree(disk_source_tree.get(),
1254 descriptor_set_in_database.get())) {
1255 return 1;
1256 }
1257
1258 error_collector.reset(
1259 new ErrorPrinter(error_format_, disk_source_tree.get()));
1260
1261 source_tree_database.reset(new SourceTreeDescriptorDatabase(
1262 disk_source_tree.get(), descriptor_set_in_database.get()));
1263 source_tree_database->RecordErrorsTo(error_collector.get());
1264
1265 descriptor_pool.reset(new DescriptorPool(
1266 source_tree_database.get(),
1267 source_tree_database->GetValidationErrorCollector()));
1268 }
1269
1270 descriptor_pool->EnforceWeakDependencies(true);
1271 if (!ParseInputFiles(descriptor_pool.get(), disk_source_tree.get(),
1272 &parsed_files)) {
1273 return 1;
1274 }
1275
1276 bool validation_error = false; // Defer exiting so we log more warnings.
1277
1278 for (auto& file : parsed_files) {
1279 google::protobuf::internal::VisitDescriptors(
1280 *file, [&](const FieldDescriptor& field) {
1281 if (HasReservedFieldNumber(&field)) {
1282 const char* error_link = nullptr;
1283 validation_error = true;
1284 std::string error;
1285 if (field.number() >= FieldDescriptor::kFirstReservedNumber &&
1286 field.number() <= FieldDescriptor::kLastReservedNumber) {
1287 error = absl::Substitute(
1288 "Field numbers $0 through $1 are reserved "
1289 "for the protocol buffer library implementation.",
1290 FieldDescriptor::kFirstReservedNumber,
1291 FieldDescriptor::kLastReservedNumber);
1292 } else {
1293 error = absl::Substitute(
1294 "Field number $0 is reserved for specific purposes.",
1295 field.number());
1296 }
1297 if (error_link) {
1298 absl::StrAppend(&error, "(See ", error_link, ")");
1299 }
1300 static_cast<DescriptorPool::ErrorCollector*>(error_collector.get())
1301 ->RecordError(field.file()->name(), field.full_name(), nullptr,
1302 DescriptorPool::ErrorCollector::NUMBER, error);
1303 }
1304 });
1305 }
1306
1307 // We visit one file at a time because we need to provide the file name for
1308 // error messages. Usually we can get the file name from any descriptor with
1309 // something like descriptor->file()->name(), but ExtensionRange does not
1310 // support this.
1311 for (const google::protobuf::FileDescriptor* file : parsed_files) {
1312 FileDescriptorProto proto;
1313 file->CopyTo(&proto);
1314 google::protobuf::internal::VisitDescriptors(
1315 *file, proto, [&](const auto& descriptor, const auto& proto) {
1316 if (!ValidateTargetConstraints(proto.options(), *descriptor_pool,
1317 *error_collector, file->name(),
1318 GetTargetType(&descriptor))) {
1319 validation_error = true;
1320 }
1321 });
1322 }
1323
1324
1325 if (validation_error) {
1326 return 1;
1327 }
1328
1329 // We construct a separate GeneratorContext for each output location. Note
1330 // that two code generators may output to the same location, in which case
1331 // they should share a single GeneratorContext so that OpenForInsert() works.
1332 GeneratorContextMap output_directories;
1333
1334 // Generate output.
1335 if (mode_ == MODE_COMPILE) {
1336 for (int i = 0; i < output_directives_.size(); i++) {
1337 std::string output_location = output_directives_[i].output_location;
1338 if (!absl::EndsWith(output_location, ".zip") &&
1339 !absl::EndsWith(output_location, ".jar") &&
1340 !absl::EndsWith(output_location, ".srcjar")) {
1341 AddTrailingSlash(&output_location);
1342 }
1343
1344 auto& generator = output_directories[output_location];
1345
1346 if (!generator) {
1347 // First time we've seen this output location.
1348 generator = std::make_unique<GeneratorContextImpl>(parsed_files);
1349 }
1350
1351 if (!GenerateOutput(parsed_files, output_directives_[i],
1352 generator.get())) {
1353 return 1;
1354 }
1355 }
1356 }
1357
1358 for (const auto& pair : output_directories) {
1359 const std::string& location = pair.first;
1360 GeneratorContextImpl* directory = pair.second.get();
1361 if (absl::EndsWith(location, "/")) {
1362 if (!directory->WriteAllToDisk(location)) {
1363 return 1;
1364 }
1365 } else {
1366 if (absl::EndsWith(location, ".jar")) {
1367 directory->AddJarManifest();
1368 }
1369
1370 if (!directory->WriteAllToZip(location)) {
1371 return 1;
1372 }
1373 }
1374 }
1375
1376 if (!dependency_out_name_.empty()) {
1377 ABSL_DCHECK(disk_source_tree.get());
1378 if (!GenerateDependencyManifestFile(parsed_files, output_directories,
1379 disk_source_tree.get())) {
1380 return 1;
1381 }
1382 }
1383
1384 if (!descriptor_set_out_name_.empty()) {
1385 if (!WriteDescriptorSet(parsed_files)) {
1386 return 1;
1387 }
1388 }
1389
1390 if (mode_ == MODE_ENCODE || mode_ == MODE_DECODE) {
1391 if (codec_type_.empty()) {
1392 // HACK: Define an EmptyMessage type to use for decoding.
1393 DescriptorPool pool;
1394 FileDescriptorProto file;
1395 file.set_name("empty_message.proto");
1396 file.add_message_type()->set_name("EmptyMessage");
1397 ABSL_CHECK(pool.BuildFile(file) != nullptr);
1398 codec_type_ = "EmptyMessage";
1399 if (!EncodeOrDecode(&pool)) {
1400 return 1;
1401 }
1402 } else {
1403 if (!EncodeOrDecode(descriptor_pool.get())) {
1404 return 1;
1405 }
1406 }
1407 }
1408
1409 if (error_collector->FoundErrors() ||
1410 (fatal_warnings_ && error_collector->FoundWarnings())) {
1411 return 1;
1412 }
1413
1414 if (mode_ == MODE_PRINT) {
1415 switch (print_mode_) {
1416 case PRINT_FREE_FIELDS:
1417 for (int i = 0; i < parsed_files.size(); ++i) {
1418 const FileDescriptor* fd = parsed_files[i];
1419 for (int j = 0; j < fd->message_type_count(); ++j) {
1420 PrintFreeFieldNumbers(fd->message_type(j));
1421 }
1422 }
1423 break;
1424 case PRINT_NONE:
1425 ABSL_LOG(ERROR)
1426 << "If the code reaches here, it usually means a bug of "
1427 "flag parsing in the CommandLineInterface.";
1428 return 1;
1429
1430 // Do not add a default case.
1431 }
1432 }
1433 return 0;
1434 }
1435
1436 bool CommandLineInterface::InitializeDiskSourceTree(
1437 DiskSourceTree* source_tree, DescriptorDatabase* fallback_database) {
1438 AddDefaultProtoPaths(&proto_path_);
1439
1440 // Set up the source tree.
1441 for (int i = 0; i < proto_path_.size(); i++) {
1442 source_tree->MapPath(proto_path_[i].first, proto_path_[i].second);
1443 }
1444
1445 // Map input files to virtual paths if possible.
1446 if (!MakeInputsBeProtoPathRelative(source_tree, fallback_database)) {
1447 return false;
1448 }
1449
1450 return true;
1451 }
1452
1453 namespace {
1454 std::unique_ptr<SimpleDescriptorDatabase>
1455 PopulateSingleSimpleDescriptorDatabase(const std::string& descriptor_set_name) {
1456 int fd;
1457 do {
1458 fd = open(descriptor_set_name.c_str(), O_RDONLY | O_BINARY);
1459 } while (fd < 0 && errno == EINTR);
1460 if (fd < 0) {
1461 std::cerr << descriptor_set_name << ": " << strerror(ENOENT) << std::endl;
1462 return nullptr;
1463 }
1464
1465 FileDescriptorSet file_descriptor_set;
1466 bool parsed = file_descriptor_set.ParseFromFileDescriptor(fd);
1467 if (close(fd) != 0) {
1468 std::cerr << descriptor_set_name << ": close: " << strerror(errno)
1469 << std::endl;
1470 return nullptr;
1471 }
1472
1473 if (!parsed) {
1474 std::cerr << descriptor_set_name << ": Unable to parse." << std::endl;
1475 return nullptr;
1476 }
1477
1478 std::unique_ptr<SimpleDescriptorDatabase> database{
1479 new SimpleDescriptorDatabase()};
1480
1481 for (int j = 0; j < file_descriptor_set.file_size(); j++) {
1482 FileDescriptorProto previously_added_file_descriptor_proto;
1483 if (database->FindFileByName(file_descriptor_set.file(j).name(),
1484 &previously_added_file_descriptor_proto)) {
1485 // already present - skip
1486 continue;
1487 }
1488 if (!database->Add(file_descriptor_set.file(j))) {
1489 return nullptr;
1490 }
1491 }
1492 return database;
1493 }
1494
1495 } // namespace
1496
1497
1498 bool CommandLineInterface::VerifyInputFilesInDescriptors(
1499 DescriptorDatabase* database) {
1500 for (const auto& input_file : input_files_) {
1501 FileDescriptorProto file_descriptor;
1502 if (!database->FindFileByName(input_file, &file_descriptor)) {
1503 std::cerr << "Could not find file in descriptor database: " << input_file
1504 << ": " << strerror(ENOENT) << std::endl;
1505 return false;
1506 }
1507
1508 // Enforce --disallow_services.
1509 if (disallow_services_ && file_descriptor.service_size() > 0) {
1510 std::cerr << file_descriptor.name()
1511 << ": This file contains services, but "
1512 "--disallow_services was used."
1513 << std::endl;
1514 return false;
1515 }
1516
1517 }
1518 return true;
1519 }
1520
1521 bool CommandLineInterface::ParseInputFiles(
1522 DescriptorPool* descriptor_pool, DiskSourceTree* source_tree,
1523 std::vector<const FileDescriptor*>* parsed_files) {
1524
1525 if (!proto_path_.empty()) {
1526 // Track unused imports in all source files that were loaded from the
1527 // filesystem. We do not track unused imports for files loaded from
1528 // descriptor sets as they may be programmatically generated in which case
1529 // exerting this level of rigor is less desirable. We're also making the
1530 // assumption that the initial parse of the proto from the filesystem
1531 // was rigorous in checking unused imports and that the descriptor set
1532 // being parsed was produced then and that it was subsequent mutations
1533 // of that descriptor set that left unused imports.
1534 //
1535 // Note that relying on proto_path exclusively is limited in that we may
1536 // be loading descriptors from both the filesystem and descriptor sets
1537 // depending on the invocation. At least for invocations that are
1538 // exclusively reading from descriptor sets, we can eliminate this failure
1539 // condition.
1540 for (const auto& input_file : input_files_) {
1541 descriptor_pool->AddUnusedImportTrackFile(input_file);
1542 }
1543 }
1544
1545 bool result = true;
1546 // Parse each file.
1547 for (const auto& input_file : input_files_) {
1548 // Import the file.
1549 const FileDescriptor* parsed_file =
1550 descriptor_pool->FindFileByName(input_file);
1551 if (parsed_file == nullptr) {
1552 result = false;
1553 break;
1554 }
1555 parsed_files->push_back(parsed_file);
1556
1557 #ifdef PROTOBUF_FUTURE_EDITIONS
1558 if (!experimental_editions_ && !IsEarlyEditionsFile(parsed_file->name())) {
1559 if (FileDescriptorLegacy(parsed_file).syntax() ==
1560 FileDescriptorLegacy::Syntax::SYNTAX_EDITIONS) {
1561 std::cerr
1562 << parsed_file->name()
1563 << ": This file uses editions, but --experimental_editions has not "
1564 "been enabled. This syntax is experimental and should be "
1565 "avoided."
1566 << std::endl;
1567 result = false;
1568 break;
1569 }
1570 }
1571 #endif // PROTOBUF_FUTURE_EDITIONS
1572
1573 // Enforce --disallow_services.
1574 if (disallow_services_ && parsed_file->service_count() > 0) {
1575 std::cerr << parsed_file->name()
1576 << ": This file contains services, but "
1577 "--disallow_services was used."
1578 << std::endl;
1579 result = false;
1580 break;
1581 }
1582
1583
1584 // Enforce --direct_dependencies
1585 if (direct_dependencies_explicitly_set_) {
1586 bool indirect_imports = false;
1587 for (int i = 0; i < parsed_file->dependency_count(); i++) {
1588 if (direct_dependencies_.find(parsed_file->dependency(i)->name()) ==
1589 direct_dependencies_.end()) {
1590 indirect_imports = true;
1591 std::cerr << parsed_file->name() << ": "
1592 << absl::StrReplaceAll(
1593 direct_dependencies_violation_msg_,
1594 {{"%s", parsed_file->dependency(i)->name()}})
1595 << std::endl;
1596 }
1597 }
1598 if (indirect_imports) {
1599 result = false;
1600 break;
1601 }
1602 }
1603 }
1604 descriptor_pool->ClearUnusedImportTrackFiles();
1605 return result;
1606 }
1607
1608 void CommandLineInterface::Clear() {
1609 // Clear all members that are set by Run(). Note that we must not clear
1610 // members which are set by other methods before Run() is called.
1611 executable_name_.clear();
1612 proto_path_.clear();
1613 input_files_.clear();
1614 direct_dependencies_.clear();
1615 direct_dependencies_violation_msg_ = kDefaultDirectDependenciesViolationMsg;
1616 output_directives_.clear();
1617 codec_type_.clear();
1618 descriptor_set_in_names_.clear();
1619 descriptor_set_out_name_.clear();
1620 dependency_out_name_.clear();
1621
1622 #ifdef PROTOBUF_FUTURE_EDITIONS
1623 experimental_editions_ = false;
1624 #endif // PROTOBUF_FUTURE_EDITIONS
1625
1626 mode_ = MODE_COMPILE;
1627 print_mode_ = PRINT_NONE;
1628 imports_in_descriptor_set_ = false;
1629 source_info_in_descriptor_set_ = false;
1630 retain_options_in_descriptor_set_ = false;
1631 disallow_services_ = false;
1632 direct_dependencies_explicitly_set_ = false;
1633 deterministic_output_ = false;
1634 }
1635
1636 bool CommandLineInterface::MakeProtoProtoPathRelative(
1637 DiskSourceTree* source_tree, std::string* proto,
1638 DescriptorDatabase* fallback_database) {
1639 // If it's in the fallback db, don't report non-existent file errors.
1640 FileDescriptorProto fallback_file;
1641 bool in_fallback_database =
1642 fallback_database != nullptr &&
1643 fallback_database->FindFileByName(*proto, &fallback_file);
1644
1645 // If the input file path is not a physical file path, it must be a virtual
1646 // path.
1647 if (access(proto->c_str(), F_OK) < 0) {
1648 std::string disk_file;
1649 if (source_tree->VirtualFileToDiskFile(*proto, &disk_file) ||
1650 in_fallback_database) {
1651 return true;
1652 } else {
1653 std::cerr << "Could not make proto path relative: " << *proto << ": "
1654 << strerror(ENOENT) << std::endl;
1655 return false;
1656 }
1657 }
1658
1659 std::string virtual_file, shadowing_disk_file;
1660 switch (source_tree->DiskFileToVirtualFile(*proto, &virtual_file,
1661 &shadowing_disk_file)) {
1662 case DiskSourceTree::SUCCESS:
1663 *proto = virtual_file;
1664 break;
1665 case DiskSourceTree::SHADOWED:
1666 std::cerr << *proto << ": Input is shadowed in the --proto_path by \""
1667 << shadowing_disk_file
1668 << "\". Either use the latter file as your input or reorder "
1669 "the --proto_path so that the former file's location "
1670 "comes first."
1671 << std::endl;
1672 return false;
1673 case DiskSourceTree::CANNOT_OPEN: {
1674 if (in_fallback_database) {
1675 return true;
1676 }
1677 std::string error_str = source_tree->GetLastErrorMessage().empty()
1678 ? strerror(errno)
1679 : source_tree->GetLastErrorMessage();
1680 std::cerr << "Could not map to virtual file: " << *proto << ": "
1681 << error_str << std::endl;
1682 return false;
1683 }
1684 case DiskSourceTree::NO_MAPPING: {
1685 // Try to interpret the path as a virtual path.
1686 std::string disk_file;
1687 if (source_tree->VirtualFileToDiskFile(*proto, &disk_file) ||
1688 in_fallback_database) {
1689 return true;
1690 } else {
1691 // The input file path can't be mapped to any --proto_path and it also
1692 // can't be interpreted as a virtual path.
1693 std::cerr
1694 << *proto
1695 << ": File does not reside within any path "
1696 "specified using --proto_path (or -I). You must specify a "
1697 "--proto_path which encompasses this file. Note that the "
1698 "proto_path must be an exact prefix of the .proto file "
1699 "names -- protoc is too dumb to figure out when two paths "
1700 "(e.g. absolute and relative) are equivalent (it's harder "
1701 "than you think)."
1702 << std::endl;
1703 return false;
1704 }
1705 }
1706 }
1707 return true;
1708 }
1709
1710 bool CommandLineInterface::MakeInputsBeProtoPathRelative(
1711 DiskSourceTree* source_tree, DescriptorDatabase* fallback_database) {
1712 for (auto& input_file : input_files_) {
1713 if (!MakeProtoProtoPathRelative(source_tree, &input_file,
1714 fallback_database)) {
1715 return false;
1716 }
1717 }
1718
1719 return true;
1720 }
1721
1722
1723 bool CommandLineInterface::ExpandArgumentFile(
1724 const std::string& file, std::vector<std::string>* arguments) {
1725 // The argument file is searched in the working directory only. We don't
1726 // use the proto import path here.
1727 std::ifstream file_stream(file.c_str());
1728 if (!file_stream.is_open()) {
1729 return false;
1730 }
1731 std::string argument;
1732 // We don't support any kind of shell expansion right now.
1733 while (std::getline(file_stream, argument)) {
1734 arguments->push_back(argument);
1735 }
1736 return true;
1737 }
1738
1739 CommandLineInterface::ParseArgumentStatus CommandLineInterface::ParseArguments(
1740 int argc, const char* const argv[]) {
1741 executable_name_ = argv[0];
1742
1743 std::vector<std::string> arguments;
1744 for (int i = 1; i < argc; ++i) {
1745 if (argv[i][0] == '@') {
1746 if (!ExpandArgumentFile(argv[i] + 1, &arguments)) {
1747 std::cerr << "Failed to open argument file: " << (argv[i] + 1)
1748 << std::endl;
1749 return PARSE_ARGUMENT_FAIL;
1750 }
1751 continue;
1752 }
1753 arguments.push_back(argv[i]);
1754 }
1755
1756 // if no arguments are given, show help
1757 if (arguments.empty()) {
1758 PrintHelpText();
1759 return PARSE_ARGUMENT_DONE_AND_EXIT; // Exit without running compiler.
1760 }
1761
1762 // Iterate through all arguments and parse them.
1763 for (int i = 0; i < arguments.size(); ++i) {
1764 std::string name, value;
1765
1766 if (ParseArgument(arguments[i].c_str(), &name, &value)) {
1767 // Returned true => Use the next argument as the flag value.
1768 if (i + 1 == arguments.size() || arguments[i + 1][0] == '-') {
1769 std::cerr << "Missing value for flag: " << name << std::endl;
1770 if (name == "--decode") {
1771 std::cerr << "To decode an unknown message, use --decode_raw."
1772 << std::endl;
1773 }
1774 return PARSE_ARGUMENT_FAIL;
1775 } else {
1776 ++i;
1777 value = arguments[i];
1778 }
1779 }
1780
1781 ParseArgumentStatus status = InterpretArgument(name, value);
1782 if (status != PARSE_ARGUMENT_DONE_AND_CONTINUE) return status;
1783 }
1784
1785 // Make sure each plugin option has a matching plugin output.
1786 bool foundUnknownPluginOption = false;
1787 for (const auto& kv : plugin_parameters_) {
1788 if (plugins_.find(kv.first) != plugins_.end()) {
1789 continue;
1790 }
1791 bool foundImplicitPlugin = false;
1792 for (const auto& d : output_directives_) {
1793 if (d.generator == nullptr) {
1794 std::string plugin_name = PluginName(plugin_prefix_, d.name);
1795 if (plugin_name == kv.first) {
1796 foundImplicitPlugin = true;
1797 break;
1798 }
1799 }
1800 }
1801 if (!foundImplicitPlugin) {
1802 std::cerr << "Unknown flag: "
1803 // strip prefix + "gen-" and add back "_opt"
1804 << "--" << kv.first.substr(plugin_prefix_.size() + 4) << "_opt"
1805 << std::endl;
1806 foundUnknownPluginOption = true;
1807 }
1808 }
1809 if (foundUnknownPluginOption) {
1810 return PARSE_ARGUMENT_FAIL;
1811 }
1812
1813 // The --proto_path & --descriptor_set_in flags both specify places to look
1814 // for proto files. If neither were given, use the current working directory.
1815 if (proto_path_.empty() && descriptor_set_in_names_.empty()) {
1816 // Don't use make_pair as the old/default standard library on Solaris
1817 // doesn't support it without explicit template parameters, which are
1818 // incompatible with C++0x's make_pair.
1819 proto_path_.push_back(std::pair<std::string, std::string>("", "."));
1820 }
1821
1822 // Check error cases that span multiple flag values.
1823 bool missing_proto_definitions = false;
1824 switch (mode_) {
1825 case MODE_COMPILE:
1826 missing_proto_definitions = input_files_.empty();
1827 break;
1828 case MODE_DECODE:
1829 // Handle --decode_raw separately, since it requires that no proto
1830 // definitions are specified.
1831 if (codec_type_.empty()) {
1832 if (!input_files_.empty() || !descriptor_set_in_names_.empty()) {
1833 std::cerr
1834 << "When using --decode_raw, no input files should be given."
1835 << std::endl;
1836 return PARSE_ARGUMENT_FAIL;
1837 }
1838 missing_proto_definitions = false;
1839 break; // only for --decode_raw
1840 }
1841 // --decode (not raw) is handled the same way as the rest of the modes.
1842 PROTOBUF_FALLTHROUGH_INTENDED;
1843 case MODE_ENCODE:
1844 case MODE_PRINT:
1845 missing_proto_definitions =
1846 input_files_.empty() && descriptor_set_in_names_.empty();
1847 break;
1848 default:
1849 ABSL_LOG(FATAL) << "Unexpected mode: " << mode_;
1850 }
1851 if (missing_proto_definitions) {
1852 std::cerr << "Missing input file." << std::endl;
1853 return PARSE_ARGUMENT_FAIL;
1854 }
1855 if (mode_ == MODE_COMPILE && output_directives_.empty() &&
1856 descriptor_set_out_name_.empty()) {
1857 std::cerr << "Missing output directives." << std::endl;
1858 return PARSE_ARGUMENT_FAIL;
1859 }
1860 if (mode_ != MODE_COMPILE && !dependency_out_name_.empty()) {
1861 std::cerr << "Can only use --dependency_out=FILE when generating code."
1862 << std::endl;
1863 return PARSE_ARGUMENT_FAIL;
1864 }
1865 if (mode_ != MODE_ENCODE && deterministic_output_) {
1866 std::cerr << "Can only use --deterministic_output with --encode."
1867 << std::endl;
1868 return PARSE_ARGUMENT_FAIL;
1869 }
1870 if (!dependency_out_name_.empty() && input_files_.size() > 1) {
1871 std::cerr
1872 << "Can only process one input file when using --dependency_out=FILE."
1873 << std::endl;
1874 return PARSE_ARGUMENT_FAIL;
1875 }
1876 if (imports_in_descriptor_set_ && descriptor_set_out_name_.empty()) {
1877 std::cerr << "--include_imports only makes sense when combined with "
1878 "--descriptor_set_out."
1879 << std::endl;
1880 }
1881 if (source_info_in_descriptor_set_ && descriptor_set_out_name_.empty()) {
1882 std::cerr << "--include_source_info only makes sense when combined with "
1883 "--descriptor_set_out."
1884 << std::endl;
1885 }
1886 if (retain_options_in_descriptor_set_ && descriptor_set_out_name_.empty()) {
1887 std::cerr << "--retain_options only makes sense when combined with "
1888 "--descriptor_set_out."
1889 << std::endl;
1890 }
1891
1892 return PARSE_ARGUMENT_DONE_AND_CONTINUE;
1893 }
1894
1895 bool CommandLineInterface::ParseArgument(const char* arg, std::string* name,
1896 std::string* value) {
1897 bool parsed_value = false;
1898
1899 if (arg[0] != '-') {
1900 // Not a flag.
1901 name->clear();
1902 parsed_value = true;
1903 *value = arg;
1904 } else if (arg[1] == '-') {
1905 // Two dashes: Multi-character name, with '=' separating name and
1906 // value.
1907 const char* equals_pos = strchr(arg, '=');
1908 if (equals_pos != nullptr) {
1909 *name = std::string(arg, equals_pos - arg);
1910 *value = equals_pos + 1;
1911 parsed_value = true;
1912 } else {
1913 *name = arg;
1914 }
1915 } else {
1916 // One dash: One-character name, all subsequent characters are the
1917 // value.
1918 if (arg[1] == '\0') {
1919 // arg is just "-". We treat this as an input file, except that at
1920 // present this will just lead to a "file not found" error.
1921 name->clear();
1922 *value = arg;
1923 parsed_value = true;
1924 } else {
1925 *name = std::string(arg, 2);
1926 *value = arg + 2;
1927 parsed_value = !value->empty();
1928 }
1929 }
1930
1931 // Need to return true iff the next arg should be used as the value for this
1932 // one, false otherwise.
1933
1934 if (parsed_value) {
1935 // We already parsed a value for this flag.
1936 return false;
1937 }
1938
1939 if (*name == "-h" || *name == "--help" || *name == "--disallow_services" ||
1940 *name == "--include_imports" || *name == "--include_source_info" ||
1941 *name == "--retain_options" || *name == "--version" ||
1942 *name == "--decode_raw" ||
1943 #ifdef PROTOBUF_FUTURE_EDITIONS
1944 *name == "--experimental_editions" ||
1945 #endif // PROTOBUF_FUTURE_EDITIONS
1946 *name == "--print_free_field_numbers" ||
1947 *name == "--experimental_allow_proto3_optional" ||
1948 *name == "--deterministic_output" || *name == "--fatal_warnings") {
1949 // HACK: These are the only flags that don't take a value.
1950 // They probably should not be hard-coded like this but for now it's
1951 // not worth doing better.
1952 return false;
1953 }
1954
1955 // Next argument is the flag value.
1956 return true;
1957 }
1958
1959 CommandLineInterface::ParseArgumentStatus
1960 CommandLineInterface::InterpretArgument(const std::string& name,
1961 const std::string& value) {
1962 if (name.empty()) {
1963 // Not a flag. Just a filename.
1964 if (value.empty()) {
1965 std::cerr
1966 << "You seem to have passed an empty string as one of the "
1967 "arguments to "
1968 << executable_name_
1969 << ". This is actually "
1970 "sort of hard to do. Congrats. Unfortunately it is not valid "
1971 "input so the program is going to die now."
1972 << std::endl;
1973 return PARSE_ARGUMENT_FAIL;
1974 }
1975
1976 #if defined(_WIN32)
1977 // On Windows, the shell (typically cmd.exe) does not expand wildcards in
1978 // file names (e.g. foo\*.proto), so we do it ourselves.
1979 switch (google::protobuf::io::win32::ExpandWildcards(
1980 value, [this](const std::string& path) {
1981 this->input_files_.push_back(path);
1982 })) {
1983 case google::protobuf::io::win32::ExpandWildcardsResult::kSuccess:
1984 break;
1985 case google::protobuf::io::win32::ExpandWildcardsResult::kErrorNoMatchingFile:
1986 // Path does not exist, is not a file, or it's longer than MAX_PATH and
1987 // long path handling is disabled.
1988 std::cerr << "Invalid file name pattern or missing input file \""
1989 << value << "\"" << std::endl;
1990 return PARSE_ARGUMENT_FAIL;
1991 default:
1992 std::cerr << "Cannot convert path \"" << value
1993 << "\" to or from Windows style" << std::endl;
1994 return PARSE_ARGUMENT_FAIL;
1995 }
1996 #else // not _WIN32
1997 // On other platforms than Windows (e.g. Linux, Mac OS) the shell (typically
1998 // Bash) expands wildcards.
1999 input_files_.push_back(value);
2000 #endif // _WIN32
2001
2002 } else if (name == "-I" || name == "--proto_path") {
2003 // Java's -classpath (and some other languages) delimits path components
2004 // with colons. Let's accept that syntax too just to make things more
2005 // intuitive.
2006 std::vector<std::string> parts = absl::StrSplit(
2007 value, absl::ByAnyChar(CommandLineInterface::kPathSeparator),
2008 absl::SkipEmpty());
2009
2010 for (int i = 0; i < parts.size(); i++) {
2011 std::string virtual_path;
2012 std::string disk_path;
2013
2014 std::string::size_type equals_pos = parts[i].find_first_of('=');
2015 if (equals_pos == std::string::npos) {
2016 virtual_path = "";
2017 disk_path = parts[i];
2018 } else {
2019 virtual_path = parts[i].substr(0, equals_pos);
2020 disk_path = parts[i].substr(equals_pos + 1);
2021 }
2022
2023 if (disk_path.empty()) {
2024 std::cerr
2025 << "--proto_path passed empty directory name. (Use \".\" for "
2026 "current directory.)"
2027 << std::endl;
2028 return PARSE_ARGUMENT_FAIL;
2029 }
2030
2031 // Make sure disk path exists, warn otherwise.
2032 if (access(disk_path.c_str(), F_OK) < 0) {
2033 // Try the original path; it may have just happened to have a '=' in it.
2034 if (access(parts[i].c_str(), F_OK) < 0) {
2035 std::cerr << disk_path << ": warning: directory does not exist."
2036 << std::endl;
2037 } else {
2038 virtual_path = "";
2039 disk_path = parts[i];
2040 }
2041 }
2042
2043 // Don't use make_pair as the old/default standard library on Solaris
2044 // doesn't support it without explicit template parameters, which are
2045 // incompatible with C++0x's make_pair.
2046 proto_path_.push_back(
2047 std::pair<std::string, std::string>(virtual_path, disk_path));
2048 }
2049
2050 } else if (name == "--direct_dependencies") {
2051 if (direct_dependencies_explicitly_set_) {
2052 std::cerr << name
2053 << " may only be passed once. To specify multiple "
2054 "direct dependencies, pass them all as a single "
2055 "parameter separated by ':'."
2056 << std::endl;
2057 return PARSE_ARGUMENT_FAIL;
2058 }
2059
2060 direct_dependencies_explicitly_set_ = true;
2061 std::vector<std::string> direct =
2062 absl::StrSplit(value, ":", absl::SkipEmpty());
2063 ABSL_DCHECK(direct_dependencies_.empty());
2064 direct_dependencies_.insert(direct.begin(), direct.end());
2065
2066 } else if (name == "--direct_dependencies_violation_msg") {
2067 direct_dependencies_violation_msg_ = value;
2068
2069 } else if (name == "--descriptor_set_in") {
2070 if (!descriptor_set_in_names_.empty()) {
2071 std::cerr << name
2072 << " may only be passed once. To specify multiple "
2073 "descriptor sets, pass them all as a single "
2074 "parameter separated by '"
2075 << CommandLineInterface::kPathSeparator << "'." << std::endl;
2076 return PARSE_ARGUMENT_FAIL;
2077 }
2078 if (value.empty()) {
2079 std::cerr << name << " requires a non-empty value." << std::endl;
2080 return PARSE_ARGUMENT_FAIL;
2081 }
2082 if (!dependency_out_name_.empty()) {
2083 std::cerr << name << " cannot be used with --dependency_out."
2084 << std::endl;
2085 return PARSE_ARGUMENT_FAIL;
2086 }
2087
2088 descriptor_set_in_names_ = absl::StrSplit(
2089 value, absl::ByAnyChar(CommandLineInterface::kPathSeparator),
2090 absl::SkipEmpty());
2091
2092 } else if (name == "-o" || name == "--descriptor_set_out") {
2093 if (!descriptor_set_out_name_.empty()) {
2094 std::cerr << name << " may only be passed once." << std::endl;
2095 return PARSE_ARGUMENT_FAIL;
2096 }
2097 if (value.empty()) {
2098 std::cerr << name << " requires a non-empty value." << std::endl;
2099 return PARSE_ARGUMENT_FAIL;
2100 }
2101 if (mode_ != MODE_COMPILE) {
2102 std::cerr
2103 << "Cannot use --encode or --decode and generate descriptors at the "
2104 "same time."
2105 << std::endl;
2106 return PARSE_ARGUMENT_FAIL;
2107 }
2108 descriptor_set_out_name_ = value;
2109
2110 } else if (name == "--dependency_out") {
2111 if (!dependency_out_name_.empty()) {
2112 std::cerr << name << " may only be passed once." << std::endl;
2113 return PARSE_ARGUMENT_FAIL;
2114 }
2115 if (value.empty()) {
2116 std::cerr << name << " requires a non-empty value." << std::endl;
2117 return PARSE_ARGUMENT_FAIL;
2118 }
2119 if (!descriptor_set_in_names_.empty()) {
2120 std::cerr << name << " cannot be used with --descriptor_set_in."
2121 << std::endl;
2122 return PARSE_ARGUMENT_FAIL;
2123 }
2124 dependency_out_name_ = value;
2125
2126 } else if (name == "--include_imports") {
2127 if (imports_in_descriptor_set_) {
2128 std::cerr << name << " may only be passed once." << std::endl;
2129 return PARSE_ARGUMENT_FAIL;
2130 }
2131 imports_in_descriptor_set_ = true;
2132
2133 } else if (name == "--include_source_info") {
2134 if (source_info_in_descriptor_set_) {
2135 std::cerr << name << " may only be passed once." << std::endl;
2136 return PARSE_ARGUMENT_FAIL;
2137 }
2138 source_info_in_descriptor_set_ = true;
2139
2140 } else if (name == "--retain_options") {
2141 if (retain_options_in_descriptor_set_) {
2142 std::cerr << name << " may only be passed once." << std::endl;
2143 return PARSE_ARGUMENT_FAIL;
2144 }
2145 retain_options_in_descriptor_set_ = true;
2146
2147 } else if (name == "-h" || name == "--help") {
2148 PrintHelpText();
2149 return PARSE_ARGUMENT_DONE_AND_EXIT; // Exit without running compiler.
2150
2151 } else if (name == "--version") {
2152 if (!version_info_.empty()) {
2153 std::cout << version_info_ << std::endl;
2154 }
2155 std::cout << "libprotoc " << internal::ProtocVersionString(PROTOBUF_VERSION)
2156 << PROTOBUF_VERSION_SUFFIX << std::endl;
2157 return PARSE_ARGUMENT_DONE_AND_EXIT; // Exit without running compiler.
2158
2159 } else if (name == "--disallow_services") {
2160 disallow_services_ = true;
2161
2162
2163 } else if (name == "--experimental_allow_proto3_optional") {
2164 // Flag is no longer observed, but we allow it for backward compat.
2165 } else if (name == "--encode" || name == "--decode" ||
2166 name == "--decode_raw") {
2167 if (mode_ != MODE_COMPILE) {
2168 std::cerr << "Only one of --encode and --decode can be specified."
2169 << std::endl;
2170 return PARSE_ARGUMENT_FAIL;
2171 }
2172 if (!output_directives_.empty() || !descriptor_set_out_name_.empty()) {
2173 std::cerr << "Cannot use " << name
2174 << " and generate code or descriptors at the same time."
2175 << std::endl;
2176 return PARSE_ARGUMENT_FAIL;
2177 }
2178
2179 mode_ = (name == "--encode") ? MODE_ENCODE : MODE_DECODE;
2180
2181 if (value.empty() && name != "--decode_raw") {
2182 std::cerr << "Type name for " << name << " cannot be blank." << std::endl;
2183 if (name == "--decode") {
2184 std::cerr << "To decode an unknown message, use --decode_raw."
2185 << std::endl;
2186 }
2187 return PARSE_ARGUMENT_FAIL;
2188 } else if (!value.empty() && name == "--decode_raw") {
2189 std::cerr << "--decode_raw does not take a parameter." << std::endl;
2190 return PARSE_ARGUMENT_FAIL;
2191 }
2192
2193 codec_type_ = value;
2194
2195 } else if (name == "--deterministic_output") {
2196 deterministic_output_ = true;
2197
2198 } else if (name == "--error_format") {
2199 if (value == "gcc") {
2200 error_format_ = ERROR_FORMAT_GCC;
2201 } else if (value == "msvs") {
2202 error_format_ = ERROR_FORMAT_MSVS;
2203 } else {
2204 std::cerr << "Unknown error format: " << value << std::endl;
2205 return PARSE_ARGUMENT_FAIL;
2206 }
2207
2208 } else if (name == "--fatal_warnings") {
2209 if (fatal_warnings_) {
2210 std::cerr << name << " may only be passed once." << std::endl;
2211 return PARSE_ARGUMENT_FAIL;
2212 }
2213 fatal_warnings_ = true;
2214 } else if (name == "--plugin") {
2215 if (plugin_prefix_.empty()) {
2216 std::cerr << "This compiler does not support plugins." << std::endl;
2217 return PARSE_ARGUMENT_FAIL;
2218 }
2219
2220 std::string plugin_name;
2221 std::string path;
2222
2223 std::string::size_type equals_pos = value.find_first_of('=');
2224 if (equals_pos == std::string::npos) {
2225 // Use the basename of the file.
2226 std::string::size_type slash_pos = value.find_last_of('/');
2227 if (slash_pos == std::string::npos) {
2228 plugin_name = value;
2229 } else {
2230 plugin_name = value.substr(slash_pos + 1);
2231 }
2232 path = value;
2233 } else {
2234 plugin_name = value.substr(0, equals_pos);
2235 path = value.substr(equals_pos + 1);
2236 }
2237
2238 plugins_[plugin_name] = path;
2239
2240 } else if (name == "--print_free_field_numbers") {
2241 if (mode_ != MODE_COMPILE) {
2242 std::cerr << "Cannot use " << name
2243 << " and use --encode, --decode or print "
2244 << "other info at the same time." << std::endl;
2245 return PARSE_ARGUMENT_FAIL;
2246 }
2247 if (!output_directives_.empty() || !descriptor_set_out_name_.empty()) {
2248 std::cerr << "Cannot use " << name
2249 << " and generate code or descriptors at the same time."
2250 << std::endl;
2251 return PARSE_ARGUMENT_FAIL;
2252 }
2253 mode_ = MODE_PRINT;
2254 print_mode_ = PRINT_FREE_FIELDS;
2255 } else if (name == "--enable_codegen_trace") {
2256 // We use environment variables here so that subprocesses see this setting
2257 // when we spawn them.
2258 //
2259 // Setting environment variables is more-or-less asking for a data race,
2260 // because C got this wrong and did not mandate synchronization.
2261 // In practice, this code path is "only" in the main thread of protoc, and
2262 // it is common knowledge that touching setenv in a library is asking for
2263 // life-ruining bugs *anyways*. As such, there is a reasonable probability
2264 // that there isn't another thread kicking environment variables at this
2265 // moment.
2266
2267 #ifdef _WIN32
2268 ::_putenv(absl::StrCat(io::Printer::kProtocCodegenTrace, "=yes").c_str());
2269 #else
2270 ::setenv(io::Printer::kProtocCodegenTrace.data(), "yes", 0);
2271 #endif
2272 #ifdef PROTOBUF_FUTURE_EDITIONS
2273 } else if (name == "--experimental_editions") {
2274 // If you're reading this, you're probably wondering what
2275 // --experimental_editions is for and thinking of turning it on. This is an
2276 // experimental, undocumented, unsupported flag. Enable it at your own risk
2277 // (or, just don't!).
2278 experimental_editions_ = true;
2279 #endif // PROTOBUF_FUTURE_EDITIONS
2280 } else {
2281 // Some other flag. Look it up in the generators list.
2282 const GeneratorInfo* generator_info = FindGeneratorByFlag(name);
2283 if (generator_info == nullptr &&
2284 (plugin_prefix_.empty() || !absl::EndsWith(name, "_out"))) {
2285 // Check if it's a generator option flag.
2286 generator_info = FindGeneratorByOption(name);
2287 if (generator_info != nullptr) {
2288 std::string* parameters =
2289 &generator_parameters_[generator_info->flag_name];
2290 if (!parameters->empty()) {
2291 parameters->append(",");
2292 }
2293 parameters->append(value);
2294 } else if (absl::StartsWith(name, "--") && absl::EndsWith(name, "_opt")) {
2295 std::string* parameters =
2296 &plugin_parameters_[PluginName(plugin_prefix_, name)];
2297 if (!parameters->empty()) {
2298 parameters->append(",");
2299 }
2300 parameters->append(value);
2301 } else {
2302 std::cerr << "Unknown flag: " << name << std::endl;
2303 return PARSE_ARGUMENT_FAIL;
2304 }
2305 } else {
2306 // It's an output flag. Add it to the output directives.
2307 if (mode_ != MODE_COMPILE) {
2308 std::cerr << "Cannot use --encode, --decode or print .proto info and "
2309 "generate code at the same time."
2310 << std::endl;
2311 return PARSE_ARGUMENT_FAIL;
2312 }
2313
2314 OutputDirective directive;
2315 directive.name = name;
2316 if (generator_info == nullptr) {
2317 directive.generator = nullptr;
2318 } else {
2319 directive.generator = generator_info->generator;
2320 }
2321
2322 // Split value at ':' to separate the generator parameter from the
2323 // filename. However, avoid doing this if the colon is part of a valid
2324 // Windows-style absolute path.
2325 std::string::size_type colon_pos = value.find_first_of(':');
2326 if (colon_pos == std::string::npos || IsWindowsAbsolutePath(value)) {
2327 directive.output_location = value;
2328 } else {
2329 directive.parameter = value.substr(0, colon_pos);
2330 directive.output_location = value.substr(colon_pos + 1);
2331 }
2332
2333 output_directives_.push_back(directive);
2334 }
2335 }
2336
2337 return PARSE_ARGUMENT_DONE_AND_CONTINUE;
2338 }
2339
2340 void CommandLineInterface::PrintHelpText() {
2341 // Sorry for indentation here; line wrapping would be uglier.
2342 std::cout << "Usage: " << executable_name_ << " [OPTION] PROTO_FILES";
2343 std::cout << R"(
2344 Parse PROTO_FILES and generate output based on the options given:
2345 -IPATH, --proto_path=PATH Specify the directory in which to search for
2346 imports. May be specified multiple times;
2347 directories will be searched in order. If not
2348 given, the current working directory is used.
2349 If not found in any of the these directories,
2350 the --descriptor_set_in descriptors will be
2351 checked for required proto file.
2352 --version Show version info and exit.
2353 -h, --help Show this text and exit.
2354 --encode=MESSAGE_TYPE Read a text-format message of the given type
2355 from standard input and write it in binary
2356 to standard output. The message type must
2357 be defined in PROTO_FILES or their imports.
2358 --deterministic_output When using --encode, ensure map fields are
2359 deterministically ordered. Note that this order
2360 is not canonical, and changes across builds or
2361 releases of protoc.
2362 --decode=MESSAGE_TYPE Read a binary message of the given type from
2363 standard input and write it in text format
2364 to standard output. The message type must
2365 be defined in PROTO_FILES or their imports.
2366 --decode_raw Read an arbitrary protocol message from
2367 standard input and write the raw tag/value
2368 pairs in text format to standard output. No
2369 PROTO_FILES should be given when using this
2370 flag.
2371 --descriptor_set_in=FILES Specifies a delimited list of FILES
2372 each containing a FileDescriptorSet (a
2373 protocol buffer defined in descriptor.proto).
2374 The FileDescriptor for each of the PROTO_FILES
2375 provided will be loaded from these
2376 FileDescriptorSets. If a FileDescriptor
2377 appears multiple times, the first occurrence
2378 will be used.
2379 -oFILE, Writes a FileDescriptorSet (a protocol buffer,
2380 --descriptor_set_out=FILE defined in descriptor.proto) containing all of
2381 the input files to FILE.
2382 --include_imports When using --descriptor_set_out, also include
2383 all dependencies of the input files in the
2384 set, so that the set is self-contained.
2385 --include_source_info When using --descriptor_set_out, do not strip
2386 SourceCodeInfo from the FileDescriptorProto.
2387 This results in vastly larger descriptors that
2388 include information about the original
2389 location of each decl in the source file as
2390 well as surrounding comments.
2391 --retain_options When using --descriptor_set_out, do not strip
2392 any options from the FileDescriptorProto.
2393 This results in potentially larger descriptors
2394 that include information about options that were
2395 only meant to be useful during compilation.
2396 --dependency_out=FILE Write a dependency output file in the format
2397 expected by make. This writes the transitive
2398 set of input file paths to FILE
2399 --error_format=FORMAT Set the format in which to print errors.
2400 FORMAT may be 'gcc' (the default) or 'msvs'
2401 (Microsoft Visual Studio format).
2402 --fatal_warnings Make warnings be fatal (similar to -Werr in
2403 gcc). This flag will make protoc return
2404 with a non-zero exit code if any warnings
2405 are generated.
2406 --print_free_field_numbers Print the free field numbers of the messages
2407 defined in the given proto files. Extension ranges
2408 are counted as occupied fields numbers.
2409 --enable_codegen_trace Enables tracing which parts of protoc are
2410 responsible for what codegen output. Not supported
2411 by all backends or on all platforms.)";
2412 if (!plugin_prefix_.empty()) {
2413 std::cout << R"(
2414 --plugin=EXECUTABLE Specifies a plugin executable to use.
2415 Normally, protoc searches the PATH for
2416 plugins, but you may specify additional
2417 executables not in the path using this flag.
2418 Additionally, EXECUTABLE may be of the form
2419 NAME=PATH, in which case the given plugin name
2420 is mapped to the given executable even if
2421 the executable's own name differs.)";
2422 }
2423
2424 for (const auto& kv : generators_by_flag_name_) {
2425 // FIXME(kenton): If the text is long enough it will wrap, which is ugly,
2426 // but fixing this nicely (e.g. splitting on spaces) is probably more
2427 // trouble than it's worth.
2428 std::cout << std::endl
2429 << " " << kv.first << "=OUT_DIR "
2430 << std::string(19 - kv.first.size(),
2431 ' ') // Spaces for alignment.
2432 << kv.second.help_text;
2433 }
2434 std::cout << R"(
2435 @<filename> Read options and filenames from file. If a
2436 relative file path is specified, the file
2437 will be searched in the working directory.
2438 The --proto_path option will not affect how
2439 this argument file is searched. Content of
2440 the file will be expanded in the position of
2441 @<filename> as in the argument list. Note
2442 that shell expansion is not applied to the
2443 content of the file (i.e., you cannot use
2444 quotes, wildcards, escapes, commands, etc.).
2445 Each line corresponds to a single argument,
2446 even if it contains spaces.)";
2447 std::cout << std::endl;
2448 }
2449
2450 bool CommandLineInterface::EnforceProto3OptionalSupport(
2451 const std::string& codegen_name, uint64_t supported_features,
2452 const std::vector<const FileDescriptor*>& parsed_files) const {
2453 bool supports_proto3_optional =
2454 supported_features & CodeGenerator::FEATURE_PROTO3_OPTIONAL;
2455 if (!supports_proto3_optional) {
2456 for (const auto fd : parsed_files) {
2457 if (ContainsProto3Optional(fd)) {
2458 std::cerr << fd->name()
2459 << ": is a proto3 file that contains optional fields, but "
2460 "code generator "
2461 << codegen_name
2462 << " hasn't been updated to support optional fields in "
2463 "proto3. Please ask the owner of this code generator to "
2464 "support proto3 optional."
2465 << std::endl;
2466 return false;
2467 }
2468 }
2469 }
2470 return true;
2471 }
2472
2473 bool CommandLineInterface::GenerateOutput(
2474 const std::vector<const FileDescriptor*>& parsed_files,
2475 const OutputDirective& output_directive,
2476 GeneratorContext* generator_context) {
2477 // Call the generator.
2478 std::string error;
2479 if (output_directive.generator == nullptr) {
2480 // This is a plugin.
2481 ABSL_CHECK(absl::StartsWith(output_directive.name, "--") &&
2482 absl::EndsWith(output_directive.name, "_out"))
2483 << "Bad name for plugin generator: " << output_directive.name;
2484
2485 std::string plugin_name = PluginName(plugin_prefix_, output_directive.name);
2486 std::string parameters = output_directive.parameter;
2487 if (!plugin_parameters_[plugin_name].empty()) {
2488 if (!parameters.empty()) {
2489 parameters.append(",");
2490 }
2491 parameters.append(plugin_parameters_[plugin_name]);
2492 }
2493 if (!GeneratePluginOutput(parsed_files, plugin_name, parameters,
2494 generator_context, &error)) {
2495 std::cerr << output_directive.name << ": " << error << std::endl;
2496 return false;
2497 }
2498 } else {
2499 // Regular generator.
2500 std::string parameters = output_directive.parameter;
2501 if (!generator_parameters_[output_directive.name].empty()) {
2502 if (!parameters.empty()) {
2503 parameters.append(",");
2504 }
2505 parameters.append(generator_parameters_[output_directive.name]);
2506 }
2507 if (!EnforceProto3OptionalSupport(
2508 output_directive.name,
2509 output_directive.generator->GetSupportedFeatures(), parsed_files)) {
2510 return false;
2511 }
2512
2513 if (!EnforceEditionsSupport(
2514 output_directive.name,
2515 output_directive.generator->GetSupportedFeatures(), parsed_files)) {
2516 return false;
2517 }
2518
2519 if (!output_directive.generator->GenerateAll(parsed_files, parameters,
2520 generator_context, &error)) {
2521 // Generator returned an error.
2522 std::cerr << output_directive.name << ": " << error << std::endl;
2523 return false;
2524 }
2525 }
2526
2527 return true;
2528 }
2529
2530 bool CommandLineInterface::GenerateDependencyManifestFile(
2531 const std::vector<const FileDescriptor*>& parsed_files,
2532 const GeneratorContextMap& output_directories,
2533 DiskSourceTree* source_tree) {
2534 FileDescriptorSet file_set;
2535
2536 absl::flat_hash_set<const FileDescriptor*> already_seen;
2537 for (int i = 0; i < parsed_files.size(); i++) {
2538 GetTransitiveDependencies(parsed_files[i], &already_seen,
2539 file_set.mutable_file());
2540 }
2541
2542 std::vector<std::string> output_filenames;
2543 for (const auto& pair : output_directories) {
2544 const std::string& location = pair.first;
2545 GeneratorContextImpl* directory = pair.second.get();
2546 std::vector<std::string> relative_output_filenames;
2547 directory->GetOutputFilenames(&relative_output_filenames);
2548 for (int i = 0; i < relative_output_filenames.size(); i++) {
2549 std::string output_filename = location + relative_output_filenames[i];
2550 if (output_filename.compare(0, 2, "./") == 0) {
2551 output_filename = output_filename.substr(2);
2552 }
2553 output_filenames.push_back(output_filename);
2554 }
2555 }
2556
2557 if (!descriptor_set_out_name_.empty()) {
2558 output_filenames.push_back(descriptor_set_out_name_);
2559 }
2560
2561 // Create the depfile, even if it will be empty.
2562 int fd;
2563 do {
2564 fd = open(dependency_out_name_.c_str(),
2565 O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666);
2566 } while (fd < 0 && errno == EINTR);
2567
2568 if (fd < 0) {
2569 perror(dependency_out_name_.c_str());
2570 return false;
2571 }
2572
2573 // Only write to the depfile if there is at least one output_filename.
2574 // Otherwise, the depfile will be malformed.
2575 if (!output_filenames.empty()) {
2576 io::FileOutputStream out(fd);
2577 io::Printer printer(&out, '$');
2578
2579 for (size_t i = 0; i < output_filenames.size(); i++) {
2580 printer.Print(output_filenames[i]);
2581 if (i == output_filenames.size() - 1) {
2582 printer.Print(":");
2583 } else {
2584 printer.Print(" \\\n");
2585 }
2586 }
2587
2588 for (int i = 0; i < file_set.file_size(); i++) {
2589 const FileDescriptorProto& file = file_set.file(i);
2590 const std::string& virtual_file = file.name();
2591 std::string disk_file;
2592 if (source_tree &&
2593 source_tree->VirtualFileToDiskFile(virtual_file, &disk_file)) {
2594 printer.Print(" $disk_file$", "disk_file", disk_file);
2595 if (i < file_set.file_size() - 1) printer.Print("\\\n");
2596 } else {
2597 std::cerr << "Unable to identify path for file " << virtual_file
2598 << std::endl;
2599 return false;
2600 }
2601 }
2602 }
2603
2604 return true;
2605 }
2606
2607 bool CommandLineInterface::GeneratePluginOutput(
2608 const std::vector<const FileDescriptor*>& parsed_files,
2609 const std::string& plugin_name, const std::string& parameter,
2610 GeneratorContext* generator_context, std::string* error) {
2611 CodeGeneratorRequest request;
2612 CodeGeneratorResponse response;
2613 std::string processed_parameter = parameter;
2614
2615
2616 // Build the request.
2617 if (!processed_parameter.empty()) {
2618 request.set_parameter(processed_parameter);
2619 }
2620
2621
2622 absl::flat_hash_set<const FileDescriptor*> already_seen;
2623 for (const FileDescriptor* file : parsed_files) {
2624 request.add_file_to_generate(file->name());
2625 GetTransitiveDependencies(file, &already_seen, request.mutable_proto_file(),
2626 {/*.include_json_name =*/true,
2627 /*.include_source_code_info =*/true,
2628 /*.retain_options =*/true});
2629 }
2630
2631 // Populate source_file_descriptors and remove source-retention options from
2632 // proto_file.
2633 ABSL_CHECK(!parsed_files.empty());
2634 const DescriptorPool* pool = parsed_files[0]->pool();
2635 absl::flat_hash_set<std::string> files_to_generate(input_files_.begin(),
2636 input_files_.end());
2637 for (FileDescriptorProto& file_proto : *request.mutable_proto_file()) {
2638 if (files_to_generate.contains(file_proto.name())) {
2639 const FileDescriptor* file = pool->FindFileByName(file_proto.name());
2640 *request.add_source_file_descriptors() = std::move(file_proto);
2641 file_proto =
2642 google::protobuf::internal::InternalFeatureHelper::GetGeneratorProto(*file);
2643 file->CopySourceCodeInfoTo(&file_proto);
2644 file->CopyJsonNameTo(&file_proto);
2645 StripSourceRetentionOptions(*file->pool(), file_proto);
2646 }
2647 }
2648
2649 google::protobuf::compiler::Version* version =
2650 request.mutable_compiler_version();
2651 version->set_major(PROTOBUF_VERSION / 1000000);
2652 version->set_minor(PROTOBUF_VERSION / 1000 % 1000);
2653 version->set_patch(PROTOBUF_VERSION % 1000);
2654 version->set_suffix(PROTOBUF_VERSION_SUFFIX);
2655
2656 // Invoke the plugin.
2657 Subprocess subprocess;
2658
2659 if (plugins_.count(plugin_name) > 0) {
2660 subprocess.Start(plugins_[plugin_name], Subprocess::EXACT_NAME);
2661 } else {
2662 subprocess.Start(plugin_name, Subprocess::SEARCH_PATH);
2663 }
2664
2665 std::string communicate_error;
2666 if (!subprocess.Communicate(request, &response, &communicate_error)) {
2667 *error = absl::Substitute("$0: $1", plugin_name, communicate_error);
2668 return false;
2669 }
2670
2671 // Write the files. We do this even if there was a generator error in order
2672 // to match the behavior of a compiled-in generator.
2673 std::unique_ptr<io::ZeroCopyOutputStream> current_output;
2674 for (int i = 0; i < response.file_size(); i++) {
2675 const CodeGeneratorResponse::File& output_file = response.file(i);
2676
2677 if (!output_file.insertion_point().empty()) {
2678 std::string filename = output_file.name();
2679 // Open a file for insert.
2680 // We reset current_output to nullptr first so that the old file is closed
2681 // before the new one is opened.
2682 current_output.reset();
2683 current_output.reset(
2684 generator_context->OpenForInsertWithGeneratedCodeInfo(
2685 filename, output_file.insertion_point(),
2686 output_file.generated_code_info()));
2687 } else if (!output_file.name().empty()) {
2688 // Starting a new file. Open it.
2689 // We reset current_output to nullptr first so that the old file is closed
2690 // before the new one is opened.
2691 current_output.reset();
2692 current_output.reset(generator_context->Open(output_file.name()));
2693 } else if (current_output == nullptr) {
2694 *error = absl::Substitute(
2695 "$0: First file chunk returned by plugin did not specify a file "
2696 "name.",
2697 plugin_name);
2698 return false;
2699 }
2700
2701 // Use CodedOutputStream for convenience; otherwise we'd need to provide
2702 // our own buffer-copying loop.
2703 io::CodedOutputStream writer(current_output.get());
2704 writer.WriteString(output_file.content());
2705 }
2706
2707 // Check for errors.
2708 if (!response.error().empty()) {
2709 // Generator returned an error.
2710 *error = response.error();
2711 return false;
2712 } else if (!EnforceProto3OptionalSupport(
2713 plugin_name, response.supported_features(), parsed_files)) {
2714 return false;
2715 } else if (!EnforceEditionsSupport(plugin_name, response.supported_features(),
2716 parsed_files)) {
2717 return false;
2718 }
2719
2720 return true;
2721 }
2722
2723 bool CommandLineInterface::EncodeOrDecode(const DescriptorPool* pool) {
2724 // Look up the type.
2725 const Descriptor* type = pool->FindMessageTypeByName(codec_type_);
2726 if (type == nullptr) {
2727 std::cerr << "Type not defined: " << codec_type_ << std::endl;
2728 return false;
2729 }
2730
2731 DynamicMessageFactory dynamic_factory(pool);
2732 std::unique_ptr<Message> message(dynamic_factory.GetPrototype(type)->New());
2733
2734 if (mode_ == MODE_ENCODE) {
2735 SetFdToTextMode(STDIN_FILENO);
2736 SetFdToBinaryMode(STDOUT_FILENO);
2737 } else {
2738 SetFdToBinaryMode(STDIN_FILENO);
2739 SetFdToTextMode(STDOUT_FILENO);
2740 }
2741
2742 io::FileInputStream in(STDIN_FILENO);
2743 io::FileOutputStream out(STDOUT_FILENO);
2744
2745 if (mode_ == MODE_ENCODE) {
2746 // Input is text.
2747 ErrorPrinter error_collector(error_format_);
2748 TextFormat::Parser parser;
2749 parser.RecordErrorsTo(&error_collector);
2750 parser.AllowPartialMessage(true);
2751
2752 if (!parser.Parse(&in, message.get())) {
2753 std::cerr << "Failed to parse input." << std::endl;
2754 return false;
2755 }
2756 } else {
2757 // Input is binary.
2758 if (!message->ParsePartialFromZeroCopyStream(&in)) {
2759 std::cerr << "Failed to parse input." << std::endl;
2760 return false;
2761 }
2762 }
2763
2764 if (!message->IsInitialized()) {
2765 std::cerr << "warning: Input message is missing required fields: "
2766 << message->InitializationErrorString() << std::endl;
2767 }
2768
2769 if (mode_ == MODE_ENCODE) {
2770 // Output is binary.
2771 io::CodedOutputStream coded_out(&out);
2772 coded_out.SetSerializationDeterministic(deterministic_output_);
2773 if (!message->SerializePartialToCodedStream(&coded_out)) {
2774 std::cerr << "output: I/O error." << std::endl;
2775 return false;
2776 }
2777 } else {
2778 // Output is text.
2779 if (!TextFormat::Print(*message, &out)) {
2780 std::cerr << "output: I/O error." << std::endl;
2781 return false;
2782 }
2783 }
2784
2785 return true;
2786 }
2787
2788 bool CommandLineInterface::WriteDescriptorSet(
2789 const std::vector<const FileDescriptor*>& parsed_files) {
2790 FileDescriptorSet file_set;
2791
2792 absl::flat_hash_set<const FileDescriptor*> already_seen;
2793 if (!imports_in_descriptor_set_) {
2794 // Since we don't want to output transitive dependencies, but we do want
2795 // things to be in dependency order, add all dependencies that aren't in
2796 // parsed_files to already_seen. This will short circuit the recursion
2797 // in GetTransitiveDependencies.
2798 absl::flat_hash_set<const FileDescriptor*> to_output;
2799 to_output.insert(parsed_files.begin(), parsed_files.end());
2800 for (int i = 0; i < parsed_files.size(); i++) {
2801 const FileDescriptor* file = parsed_files[i];
2802 for (int j = 0; j < file->dependency_count(); j++) {
2803 const FileDescriptor* dependency = file->dependency(j);
2804 // if the dependency isn't in parsed files, mark it as already seen
2805 if (to_output.find(dependency) == to_output.end()) {
2806 already_seen.insert(dependency);
2807 }
2808 }
2809 }
2810 }
2811 TransitiveDependencyOptions options;
2812 options.include_json_name = true;
2813 options.include_source_code_info = source_info_in_descriptor_set_;
2814 options.retain_options = retain_options_in_descriptor_set_;
2815 for (int i = 0; i < parsed_files.size(); i++) {
2816 GetTransitiveDependencies(parsed_files[i], &already_seen,
2817 file_set.mutable_file(), options);
2818 }
2819
2820 int fd;
2821 do {
2822 fd = open(descriptor_set_out_name_.c_str(),
2823 O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666);
2824 } while (fd < 0 && errno == EINTR);
2825
2826 if (fd < 0) {
2827 perror(descriptor_set_out_name_.c_str());
2828 return false;
2829 }
2830
2831 io::FileOutputStream out(fd);
2832
2833 {
2834 io::CodedOutputStream coded_out(&out);
2835 // Determinism is useful here because build outputs are sometimes checked
2836 // into version control.
2837 coded_out.SetSerializationDeterministic(true);
2838 if (!file_set.SerializeToCodedStream(&coded_out)) {
2839 std::cerr << descriptor_set_out_name_ << ": " << strerror(out.GetErrno())
2840 << std::endl;
2841 out.Close();
2842 return false;
2843 }
2844 }
2845
2846 if (!out.Close()) {
2847 std::cerr << descriptor_set_out_name_ << ": " << strerror(out.GetErrno())
2848 << std::endl;
2849 return false;
2850 }
2851
2852 return true;
2853 }
2854
2855 const CommandLineInterface::GeneratorInfo*
2856 CommandLineInterface::FindGeneratorByFlag(const std::string& name) const {
2857 auto it = generators_by_flag_name_.find(name);
2858 if (it == generators_by_flag_name_.end()) return nullptr;
2859 return &it->second;
2860 }
2861
2862 const CommandLineInterface::GeneratorInfo*
2863 CommandLineInterface::FindGeneratorByOption(const std::string& option) const {
2864 auto it = generators_by_option_name_.find(option);
2865 if (it == generators_by_option_name_.end()) return nullptr;
2866 return &it->second;
2867 }
2868
2869 namespace {
2870
2871 // Utility function for PrintFreeFieldNumbers.
2872 // Stores occupied ranges into the ranges parameter, and next level of sub
2873 // message types into the nested_messages parameter. The FieldRange is left
2874 // inclusive, right exclusive. i.e. [a, b).
2875 //
2876 // Nested Messages:
2877 // Note that it only stores the nested message type, iff the nested type is
2878 // either a direct child of the given descriptor, or the nested type is a
2879 // descendant of the given descriptor and all the nodes between the
2880 // nested type and the given descriptor are group types. e.g.
2881 //
2882 // message Foo {
2883 // message Bar {
2884 // message NestedBar {}
2885 // }
2886 // group Baz = 1 {
2887 // group NestedBazGroup = 2 {
2888 // message Quz {
2889 // message NestedQuz {}
2890 // }
2891 // }
2892 // message NestedBaz {}
2893 // }
2894 // }
2895 //
2896 // In this case, Bar, Quz and NestedBaz will be added into the nested types.
2897 // Since free field numbers of group types will not be printed, this makes sure
2898 // the nested message types in groups will not be dropped. The nested_messages
2899 // parameter will contain the direct children (when groups are ignored in the
2900 // tree) of the given descriptor for the caller to traverse. The declaration
2901 // order of the nested messages is also preserved.
2902 typedef std::pair<int, int> FieldRange;
2903 void GatherOccupiedFieldRanges(
2904 const Descriptor* descriptor, absl::btree_set<FieldRange>* ranges,
2905 std::vector<const Descriptor*>* nested_messages) {
2906 for (int i = 0; i < descriptor->field_count(); ++i) {
2907 const FieldDescriptor* fd = descriptor->field(i);
2908 ranges->insert(FieldRange(fd->number(), fd->number() + 1));
2909 }
2910 for (int i = 0; i < descriptor->extension_range_count(); ++i) {
2911 ranges->insert(FieldRange(descriptor->extension_range(i)->start_number(),
2912 descriptor->extension_range(i)->end_number()));
2913 }
2914 for (int i = 0; i < descriptor->reserved_range_count(); ++i) {
2915 ranges->insert(FieldRange(descriptor->reserved_range(i)->start,
2916 descriptor->reserved_range(i)->end));
2917 }
2918 // Handle the nested messages/groups in declaration order to make it
2919 // post-order strict.
2920 for (int i = 0; i < descriptor->nested_type_count(); ++i) {
2921 const Descriptor* nested_desc = descriptor->nested_type(i);
2922 nested_messages->push_back(nested_desc);
2923 }
2924 }
2925
2926 // Utility function for PrintFreeFieldNumbers.
2927 // Actually prints the formatted free field numbers for given message name and
2928 // occupied ranges.
2929 void FormatFreeFieldNumbers(const std::string& name,
2930 const absl::btree_set<FieldRange>& ranges) {
2931 std::string output;
2932 absl::StrAppendFormat(&output, "%-35s free:", name.c_str());
2933 int next_free_number = 1;
2934 for (const auto& range : ranges) {
2935 // This happens when groups re-use parent field numbers, in which
2936 // case we skip the FieldRange entirely.
2937 if (next_free_number >= range.second) continue;
2938
2939 if (next_free_number < range.first) {
2940 if (next_free_number + 1 == range.first) {
2941 // Singleton
2942 absl::StrAppendFormat(&output, " %d", next_free_number);
2943 } else {
2944 // Range
2945 absl::StrAppendFormat(&output, " %d-%d", next_free_number,
2946 range.first - 1);
2947 }
2948 }
2949 next_free_number = range.second;
2950 }
2951 if (next_free_number <= FieldDescriptor::kMaxNumber) {
2952 absl::StrAppendFormat(&output, " %d-INF", next_free_number);
2953 }
2954 std::cout << output << std::endl;
2955 }
2956
2957 } // namespace
2958
2959 void CommandLineInterface::PrintFreeFieldNumbers(const Descriptor* descriptor) {
2960 absl::btree_set<FieldRange> ranges;
2961 std::vector<const Descriptor*> nested_messages;
2962 GatherOccupiedFieldRanges(descriptor, &ranges, &nested_messages);
2963
2964 for (int i = 0; i < nested_messages.size(); ++i) {
2965 PrintFreeFieldNumbers(nested_messages[i]);
2966 }
2967 FormatFreeFieldNumbers(descriptor->full_name(), ranges);
2968 }
2969
2970
2971 } // namespace compiler
2972 } // namespace protobuf
2973 } // namespace google