Debian Codegraph @

internal/gapicgen/generator/genproto.go

raw
1 // Copyright 2019 Google LLC
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 package generator
16
17 import (
18 "context"
19 "errors"
20 "fmt"
21 "log"
22 "os"
23 "path/filepath"
24 "regexp"
25 "strconv"
26 "strings"
27
28 "cloud.google.com/go/internal/gapicgen/execv"
29 "cloud.google.com/go/internal/gapicgen/execv/gocmd"
30 "cloud.google.com/go/internal/gapicgen/git"
31 "golang.org/x/sync/errgroup"
32 )
33
34 var goPkgOptRe = regexp.MustCompile(`(?m)^option go_package = (.*);`)
35 var ErrNoProcessing = errors.New("there are not files to regenerate")
36
37 // GenprotoGenerator is used to generate code for googleapis/go-genproto.
38 type GenprotoGenerator struct {
39 genprotoDir string
40 googleapisDir string
41 protoSrcDir string
42 forceAll bool
43 }
44
45 // NewGenprotoGenerator creates a new GenprotoGenerator.
46 func NewGenprotoGenerator(c *Config) *GenprotoGenerator {
47 return &GenprotoGenerator{
48 genprotoDir: c.GenprotoDir,
49 googleapisDir: c.GoogleapisDir,
50 protoSrcDir: filepath.Join(c.ProtoDir, "/src"),
51 forceAll: c.ForceAll,
52 }
53 }
54
55 // Only the packages listed here are generated into go-genproto.
56 var generateList = []string{
57 "google.golang.org/genproto/googleapis/type/expr",
58 "google.golang.org/genproto/googleapis/rpc/http",
59 "google.golang.org/genproto/googleapis/type/latlng",
60 "google.golang.org/genproto/googleapis/genomics/v1alpha2",
61 "google.golang.org/genproto/googleapis/type/date",
62 "google.golang.org/genproto/googleapis/type/date_time_range",
63 "google.golang.org/genproto/googleapis/api/metric",
64 "google.golang.org/genproto/googleapis/api/distribution",
65 "google.golang.org/genproto/googleapis/chromeos/moblab/v1beta1",
66 "google.golang.org/genproto/googleapis/apps/script/type/slides",
67 "google.golang.org/genproto/googleapis/api/expr/v1beta1",
68 "google.golang.org/genproto/googleapis/apps/script/type/gmail",
69 "google.golang.org/genproto/googleapis/type/month",
70 "google.golang.org/genproto/googleapis/actions/sdk/v2/interactionmodel/type",
71 "google.golang.org/genproto/googleapis/apps/alertcenter/v1beta1",
72 "google.golang.org/genproto/googleapis/api/error_reason",
73 "google.golang.org/genproto/googleapis/assistant/embedded/v1alpha1",
74 "google.golang.org/genproto/googleapis/type/localized_text",
75 "google.golang.org/genproto/googleapis/type/interval",
76 "google.golang.org/genproto/googleapis/watcher/v1",
77 "google.golang.org/genproto/googleapis/apps/script/type/docs",
78 "google.golang.org/genproto/googleapis/api/monitoredres",
79 "google.golang.org/genproto/googleapis/actions/sdk/v2/interactionmodel",
80 "google.golang.org/genproto/googleapis/type/dayofweek",
81 "google.golang.org/genproto/googleapis/gapic/metadata",
82 "google.golang.org/genproto/googleapis/chat/logging/v1",
83 "google.golang.org/genproto/googleapis/api/expr/v1alpha1",
84 "google.golang.org/genproto/googleapis/grafeas/v1",
85 "google.golang.org/genproto/googleapis/type/quaternion",
86 "google.golang.org/genproto/googleapis/type/calendarperiod",
87 "google.golang.org/genproto/googleapis/type/date_range",
88 "google.golang.org/genproto/googleapis/rpc/status",
89 "google.golang.org/genproto/googleapis/rpc/context",
90 "google.golang.org/genproto/googleapis/rpc/code",
91 "google.golang.org/genproto/googleapis/api/visibility",
92 "google.golang.org/genproto/googleapis/streetview/publish/v1",
93 "google.golang.org/genproto/googleapis/type/money",
94 "google.golang.org/genproto/googleapis/type/decimal",
95 "google.golang.org/genproto/googleapis/type/color",
96 "google.golang.org/genproto/googleapis/apps/drive/activity/v2",
97 "google.golang.org/genproto/googleapis/apps/script/type/sheets",
98 "google.golang.org/genproto/googleapis/type/timeofday",
99 "google.golang.org/genproto/googleapis/home/graph/v1",
100 "google.golang.org/genproto/googleapis/container/v1alpha1",
101 "google.golang.org/genproto/googleapis/rpc/errdetails",
102 "google.golang.org/genproto/googleapis/actions/sdk/v2",
103 "google.golang.org/genproto/googleapis/networking/trafficdirector/type",
104 "google.golang.org/genproto/googleapis/actions/sdk/v2/conversation",
105 "google.golang.org/genproto/googleapis/home/enterprise/sdm/v1",
106 "google.golang.org/genproto/googleapis/bytestream",
107 "google.golang.org/genproto/googleapis/api",
108 "google.golang.org/genproto/googleapis/apps/script/type",
109 "google.golang.org/genproto/googleapis/api/configchange",
110 "google.golang.org/genproto/googleapis/ccc/hosted/marketplace/v2",
111 "google.golang.org/genproto/googleapis/chromeos/uidetection/v1",
112 "google.golang.org/genproto/googleapis/type/datetime",
113 "google.golang.org/genproto/googleapis/geo/type/viewport",
114 "google.golang.org/genproto/googleapis/type/phone_number",
115 "google.golang.org/genproto/googleapis/type/fraction",
116 "google.golang.org/genproto/googleapis/apps/drive/labels/v2",
117 "google.golang.org/genproto/googleapis/example/library/v1",
118 "google.golang.org/genproto/googleapis/api/label",
119 "google.golang.org/genproto/googleapis/api/httpbody",
120 "google.golang.org/genproto/googleapis/partner/aistreams/v1alpha1",
121 "google.golang.org/genproto/googleapis/apps/script/type/drive",
122 "google.golang.org/genproto/googleapis/search/partnerdataingestion/logging/v1",
123 "google.golang.org/genproto/googleapis/apps/script/type/calendar",
124 "google.golang.org/genproto/googleapis/rpc/context/attribute_context",
125 "google.golang.org/genproto/googleapis/api/expr/conformance/v1alpha1",
126 "google.golang.org/genproto/googleapis/actions/sdk/v2/interactionmodel/prompt",
127 "google.golang.org/genproto/googleapis/api/serviceconfig",
128 "google.golang.org/genproto/googleapis/apps/drive/labels/v2beta",
129 "google.golang.org/genproto/googleapis/genomics/v1",
130 "google.golang.org/genproto/googleapis/api/annotations",
131 "google.golang.org/genproto/googleapis/type/postaladdress",
132 "google.golang.org/genproto/googleapis/firebase/fcm/connection/v1alpha1",
133 "google.golang.org/genproto/googleapis/assistant/embedded/v1alpha2",
134 "google.golang.org/genproto/googleapis/apps/card/v1",
135 "google.golang.org/genproto/googleapis/cloud/audit",
136 }
137
138 // Regen regenerates the genproto repository.
139 // regenGenproto regenerates the genproto repository.
140 //
141 // regenGenproto recursively walks through each directory named by given
142 // arguments, looking for all .proto files. (Symlinks are not followed.) Any
143 // proto file without `go_package` option or whose option does not begin with
144 // the genproto prefix is ignored.
145 //
146 // If multiple roots contain files with the same name, eg "root1/path/to/file"
147 // and "root2/path/to/file", only the first file is processed; the rest are
148 // ignored.
149 //
150 // Protoc is executed on remaining files, one invocation per set of files
151 // declaring the same Go package.
152 func (g *GenprotoGenerator) Regen(ctx context.Context) error {
153 log.Println("regenerating genproto")
154 // Create space to put generated .pb.go's.
155 c := execv.Command("mkdir", "-p", "generated")
156 c.Dir = g.genprotoDir
157 if err := c.Run(); err != nil {
158 return err
159 }
160
161 // Get the last processed googleapis hash.
162 lastHash, err := os.ReadFile(filepath.Join(g.genprotoDir, "regen.txt"))
163 if err != nil {
164 return err
165 }
166
167 // TODO(noahdietz): In local mode, since it clones a shallow copy with 1 commit,
168 // if the last regenerated hash is earlier than the top commit, the git diff-tree
169 // command fails. This is is a bit of a rough edge. Using my local clone of
170 // googleapis rectified the issue.
171 pkgFiles, err := g.getUpdatedPackages(string(lastHash))
172 if err != nil {
173 return err
174 }
175 pkgFiles, err = filterPackages(pkgFiles)
176 if err != nil {
177 return err
178 }
179
180 log.Println("generating from protos")
181 grp, _ := errgroup.WithContext(ctx)
182 for pkg, fileNames := range pkgFiles {
183 pk := pkg
184 fn := fileNames
185
186 grp.Go(func() error {
187 log.Println("running protoc on", pk)
188 return g.protoc(fn)
189 })
190 }
191 if err := grp.Wait(); err != nil {
192 return err
193 }
194
195 if err := g.moveAndCleanupGeneratedSrc(); err != nil {
196 return err
197 }
198
199 if err := gocmd.Vet(g.genprotoDir); err != nil {
200 return err
201 }
202
203 if err := gocmd.Build(g.genprotoDir); err != nil {
204 return err
205 }
206
207 return nil
208 }
209
210 func filterPackages(in map[string][]string) (map[string][]string, error) {
211 out := map[string][]string{}
212 for _, allowed := range generateList {
213 if files, present := in[allowed]; present {
214 out[allowed] = files
215 }
216 }
217 if len(out) == 0 {
218 return nil, ErrNoProcessing
219 }
220 return out, nil
221 }
222
223 // goPkg reports the import path declared in the given file's `go_package`
224 // option. If the option is missing, goPkg returns empty string.
225 func goPkg(fileName string) (string, error) {
226 content, err := os.ReadFile(fileName)
227 if err != nil {
228 return "", err
229 }
230
231 var pkgName string
232 if match := goPkgOptRe.FindSubmatch(content); len(match) > 0 {
233 pn, err := strconv.Unquote(string(match[1]))
234 if err != nil {
235 return "", err
236 }
237 pkgName = pn
238 }
239 if p := strings.IndexRune(pkgName, ';'); p > 0 {
240 pkgName = pkgName[:p]
241 }
242 return pkgName, nil
243 }
244
245 // protoc executes the "protoc" command on files named in fileNames, and outputs
246 // to "<genprotoDir>/generated".
247 func (g *GenprotoGenerator) protoc(fileNames []string) error {
248 args := []string{
249 "--experimental_allow_proto3_optional",
250 fmt.Sprintf("--go_out=plugins=grpc:%s/generated", g.genprotoDir),
251 "-I", g.googleapisDir,
252 "-I", g.protoSrcDir,
253 }
254 args = append(args, fileNames...)
255 c := execv.Command("protoc", args...)
256 c.Dir = g.genprotoDir
257 return c.Run()
258 }
259
260 // getUpdatedPackages parses all of the new commits to find what packages need
261 // to be regenerated.
262 func (g *GenprotoGenerator) getUpdatedPackages(googleapisHash string) (map[string][]string, error) {
263 if g.forceAll {
264 return g.getAllPackages()
265 }
266 files, err := git.UpdateFilesSinceHash(g.googleapisDir, googleapisHash)
267 if err != nil {
268 return nil, err
269 }
270 pkgFiles := make(map[string][]string)
271 for _, v := range files {
272 if !strings.HasSuffix(v, ".proto") {
273 continue
274 }
275 if strings.HasSuffix(v, "compute_small.proto") {
276 continue
277 }
278 path := filepath.Join(g.googleapisDir, v)
279 pkg, err := goPkg(path)
280 if err != nil {
281 return nil, err
282 }
283 pkgFiles[pkg] = append(pkgFiles[pkg], path)
284 }
285 return pkgFiles, nil
286 }
287
288 func (g *GenprotoGenerator) getAllPackages() (map[string][]string, error) {
289 seenFiles := make(map[string]bool)
290 pkgFiles := make(map[string][]string)
291 for _, root := range []string{g.googleapisDir} {
292 walkFn := func(path string, info os.FileInfo, err error) error {
293 if err != nil {
294 return err
295 }
296 if !info.Mode().IsRegular() || !strings.HasSuffix(path, ".proto") {
297 return nil
298 }
299
300 switch rel, err := filepath.Rel(root, path); {
301 case err != nil:
302 return err
303 case seenFiles[rel]:
304 return nil
305 default:
306 seenFiles[rel] = true
307 }
308
309 pkg, err := goPkg(path)
310 if err != nil {
311 return err
312 }
313 pkgFiles[pkg] = append(pkgFiles[pkg], path)
314 return nil
315 }
316 if err := filepath.Walk(root, walkFn); err != nil {
317 return nil, err
318 }
319 }
320 return pkgFiles, nil
321 }
322
323 // moveAndCleanupGeneratedSrc moves all generated src to their correct locations
324 // in the repository, because protoc puts it in a folder called `generated/“.
325 func (g *GenprotoGenerator) moveAndCleanupGeneratedSrc() error {
326 log.Println("moving generated code")
327 // The period at the end is analogous to * (copy everything in this dir).
328 c := execv.Command("cp", "-R", filepath.Join(g.genprotoDir, "generated", "google.golang.org", "genproto", "googleapis"), g.genprotoDir)
329 if err := c.Run(); err != nil {
330 return err
331 }
332
333 c = execv.Command("rm", "-rf", "generated")
334 c.Dir = g.genprotoDir
335 if err := c.Run(); err != nil {
336 return err
337 }
338
339 return nil
340 }