Debian Codegraph @

texmf-dist/source/latex/proposal/bin/generate-issues

raw
1 #!/usr/bin/env ruby
2 # Generate GitHub Issues for tasks and deliverables
3 # - Each work package gets a WPX/README.md file
4 # - Each task gets an Issue
5 # - Each deliverable gets a Milestone
6 # - Each deliverable gets an Issue associated with its Milestone
7 # - Each deliverable Issue gets 'deliverable', 'WPX' tags
8 # - Each task Issue gets 'task', 'WPX' tags
9
10 # Run me from the proposal dir containing proposal.pdata:
11 # cd ~/OpenDreamKit/Proposal
12 # ~/path/to/LaTeX-Proposal/bin/generate-issues
13
14 require 'date'
15 require 'Octokit'
16 require 'netrc'
17
18 # constants for running the script are for OpenDreamKit
19 # update these for your own proposal
20
21 START_DATE = Date::new(2015, 9, 1) # start date of the project
22 HOMEPAGE = "http://opendreamkit.org"
23 PROPOSAL_URL = "https://github.com/OpenDreamKit/OpenDreamKit/raw/master/Proposal/proposal-www.pdf"
24 PROJECT = "OpenDreamKit"
25 REPO = "OpenDreamKit/OpenDreamKit"
26 TARGET = "final" # or 'proposal'
27
28 # throttle github creation requests to 5 Hz to avoid getting flagged for abuse
29 THROTTLE_SECONDS = 5
30
31 NATURES = {
32 'R' => 'Report',
33 'DEM' => 'Demonstrator',
34 'DEC' => 'Websites, Media, etc.',
35 'OTHER' => 'Other',
36 }
37
38 DISSEMINATIONS = {
39 'PU' => 'Public',
40 'CO' => 'Confidential',
41 'CI' => 'Classified',
42 }
43
44 SITES = {
45 'PS' => 'Université Paris-Sud',
46 'LL' => 'Logilab',
47 'UV' => 'Université de Versailles Saint-Quentin',
48 'UJF' => 'Université Joseph Fourier',
49 'UB' => 'CNRS',
50 'UO' => 'University of Oxford',
51 'USH' => 'University of Sheffield',
52 'USO' => 'University of Southampton',
53 'SA' => 'University of St Andrews',
54 'UW' => 'University of Warwick',
55 'JU' => 'Jacobs University Bremen',
56 'UK' => 'University of Kaiserslautern',
57 'US' => 'University of Silesia',
58 'ZH' => 'Universität Zürich',
59 'SR' => 'Simula Research Laboratory',
60 }
61
62 REVERSE_SITES = {}
63 SITES.each_pair do |key, name|
64 REVERSE_SITES[name] = key
65 end
66
67 # Some sites would prefer alternative abbreviations
68 REVERSE_SITES['University of St Andrews'] = 'USTAN'
69 REVERSE_SITES['Université de Versailles Saint-Quentin'] = 'UVSQ'
70
71 #------------------- Parsing proposal.pdata ---------------------
72
73 def split_line(line)
74 # super primitive state-machine line split (not going to regex this)
75 parts = []
76 level = 0
77 buffer = []
78 line.each_char do |c|
79 case c
80 when '{'
81 if level > 0
82 buffer.push c
83 end
84 level += 1
85 when '}'
86 level -= 1
87 if level > 0
88 buffer.push c
89 end
90 if level == 0
91 parts.push(scrub_tex(buffer.join('')))
92 buffer = []
93 end
94 else
95 if level > 0
96 buffer.push c
97 end
98 end
99 end
100 return parts
101 end
102
103
104 def scrub_tex(text)
105 # scrub some latex markup from text
106 text.gsub!(/\\\w+/, '')
107 text.gsub!(/[{}]/, '')
108 text.strip.split.join(' ')
109 end
110
111
112 def transform_value(key, value)
113 case key
114 when 'lead'
115 return SITES[value]
116 when 'partners'
117 return value.split(',').map { |v| SITES[v] }
118 when 'dissem'
119 return DISSEMINATIONS[value]
120 when 'nature'
121 return NATURES[value]
122 when 'month'
123 return value.to_i
124 when 'delivs'
125 return value.split(',')
126 else
127 return scrub_tex(value)
128 end
129 end
130
131
132 def load_pdata(proposal_dir)
133 pdata = File.join(proposal_dir, "#{TARGET}.pdata")
134 deliv_data = File.join(proposal_dir, "#{TARGET}.deliverables")
135
136 workpackages = {} # mapping of workpackage id => wp info
137 deliverables = {}
138
139 File.readlines(pdata).each do |line|
140 key, *args = split_line line
141
142 case key
143 when 'wp'
144 name, key, value = args
145 value = transform_value(key, value)
146 if not workpackages.include? name
147 workpackages[name] = {
148 "tasks" => {},
149 "unknown-task" => nil,
150 "deliverables" => [],
151 }
152 end
153 wp = workpackages[name]
154 wp[key] = value
155
156 when 'task'
157 name, key, value = args
158 value = transform_value(key, value)
159 # find my workpackage
160 if name.index('@')
161 wpkey, short_name = name.split('@')
162 if short_name.match(/task\d+/)
163 workpackages[wpkey]['unknown-task'] = short_name
164 name = short_name
165 end
166 else
167 wpkey = workpackages.keys.select { |wpkey| name.start_with? wpkey }.first
168 end
169 # handle workpackage@taskNN weirdness
170 wp = workpackages[wpkey]
171 tasks = wp['tasks']
172 if not wp['unknown-task'].nil? and wp['unknown-task'] != name
173 tasks[name] = tasks.delete(wp['unknown-task'])
174 wp['unknown-task'] = nil
175 end
176 if not tasks.include? name
177 tasks[name] = {}
178 end
179 tasks[name][key] = value
180 when 'deliv'
181 name, key, value = args
182
183 value = transform_value(key, value)
184 if not deliverables.include? name
185 deliverables[name] = {}
186 end
187 deliverables[name][key] = value
188
189 else
190 # DEBUG:
191 # puts " Ignored: #{args}"
192 end
193 end
194
195 # get deliverable data, workpackage id from proposal.deliverables
196 File.readlines(deliv_data).each do |line|
197 args = split_line line
198 name = args[3]
199 if deliverables.include? name
200 deliverable = deliverables[name]
201 else
202 deliverable = {}
203 end
204 month = args[0].to_i
205 deliverable = (deliverables[name] or {}).merge({
206 "month" => month,
207 # due date is last day of the given month, so subtract one day
208 "due_date" => (START_DATE >> month) - 1,
209 "label" => scrub_tex(args[2]),
210 "deliv_id" => args[3],
211 "dissem" => transform_value('dissem', args[4]),
212 "nature" => transform_value('nature', args[5]),
213 "title" => scrub_tex(args[6]),
214 "lead" => SITES[scrub_tex(args[8])],
215 })
216 wpid = scrub_tex(args[7])
217 wp = workpackages.values.find {|wp| wp['label'] == wpid}
218 wp['deliverables'].push(deliverable)
219 end
220
221 workpackages.values.each do |wp|
222 wp['deliverables'].sort_by! {|d| d['label']}
223 end
224
225 return workpackages.values.sort_by {|wp| wp['label']}
226 end
227
228
229 #--------------- GitHub-related ---------------------
230
231 def check_token
232 # get GitHub auth token, creating one if we don't find it.
233 rc = Netrc.read Netrc.default_path
234 if not rc['api.github.com'].nil?
235 return
236 end
237 puts "We need your password to generate an OAuth token. The password will not be stored."
238 username = ask "Username: "
239 password = ask("Password: ") { |q| q.echo = '*' }
240 client = Octokit::Client.new(
241 :login => username,
242 :password => password,
243 )
244 reply = client.create_authorization(
245 :scopes => ["public_repo"],
246 :note => "Issue Migration",
247 )
248 token = reply.token
249 rc['api.github.com'] = username, token
250 rc.save
251 end
252
253 $cache = {
254 'issues' => {},
255 'milestones' => {},
256 }
257
258 def get_issues(github, repo)
259 # get issues for a repo (cached)
260 cache = $cache['issues']
261 if not cache.include? repo
262 cache[repo] = github.issues(repo)
263 end
264 return cache[repo]
265 end
266
267 def get_milestones(github, repo)
268 # get issues for a repo (cached)
269 cache = $cache['milestones']
270 if not cache.include? repo
271 cache[repo] = github.list_milestones(repo)
272 end
273 return cache[repo]
274 end
275
276 # Templates for making readmes, issues
277
278 README_TPL = <<-END
279 # %{title}
280
281 Lead Institution: %{lead}
282
283 See page %{page} of the [proposal](#{PROPOSAL_URL}) for the full description.
284 END
285
286 TASK_TPL = <<-END
287 - **%{wplabel}:** [%{wptitle}](https://github.com/#{REPO}/tree/master/%{wplabel})
288 - **Lead Institution:** %{lead}
289 - **Partners:** %{partners}
290 - **Work phases:** %{wphases}
291
292 See page %{page} of the [proposal](#{PROPOSAL_URL}) for the full description.
293 END
294
295 DELIV_TPL = <<-END
296 - **%{wplabel}:** [%{wptitle}](https://github.com/#{REPO}/tree/master/%{wplabel})
297 - **Lead Institution:** %{lead}
298 - **Due:** %{date} (month %{month})
299 - **Nature:** %{nature}
300
301 See page %{page} of the [proposal](#{PROPOSAL_URL}) for the full description.
302 END
303
304 DELIV_MILESTONE_TPL = "# %{title}\n\n#{DELIV_TPL}"
305
306
307 def make_task_issue(github, repo, task, workpackage, options)
308 title = "#{task['label']}: #{task['title']}"
309 issues = get_issues(github, repo)
310 issue = issues.find { |i| i.title.start_with?(task['label'] + ':') }
311 if issue.nil?
312 body = TASK_TPL % {
313 wplabel: workpackage['label'],
314 wptitle: workpackage['title'],
315 lead: task['lead'],
316 page: task['page'],
317 wphases: task['wphases'],
318 partners: (task['partners'] or ['None']).join(', ')
319 }
320 puts "\n\nMaking Issue on #{repo}: #{title}"
321 puts body
322
323 github.create_issue(repo, title, body, options)
324 # throttle creation calls to avoid flags for abuse
325 sleep THROTTLE_SECONDS
326 else
327 puts "Found Issue #{repo}##{issue.number}: #{issue.title}"
328 existing_labels = issue.labels.map { |label| label['name']}
329 missing_labels = options[:labels].reject { |label| existing_labels.include? label }
330 if not missing_labels.empty?
331 puts "Updating milestone, labels on #{repo}##{issue.number}"
332 github.update_issue(repo, issue.number, options)
333 sleep THROTTLE_SECONDS
334 end
335 end
336 end
337
338
339 def make_deliverable_issue(github, repo, deliverable, workpackage, options)
340 title = "#{deliverable['label']}: #{deliverable['title']}"
341 issues = get_issues(github, repo)
342 issue = issues.find { |i| i.title.start_with?(deliverable['label'] + ':') }
343 if issue.nil?
344 body = DELIV_TPL % {
345 wplabel: workpackage['label'],
346 wptitle: workpackage['title'],
347 lead: deliverable['lead'],
348 date: deliverable['due_date'],
349 month: deliverable['month'],
350 nature: deliverable['nature'],
351 page: deliverable['page'],
352 }
353 puts "\n\nMaking Issue on #{repo}: #{title}"
354 puts body
355 github.create_issue(repo, title, body, options)
356 # throttle creation calls to avoid flags for abuse
357 sleep THROTTLE_SECONDS
358 else
359 puts "Found Issue #{repo}##{issue.number}: #{issue.title}"
360 existing_labels = issue.labels.map { |label| label['name']}
361 missing_labels = options[:labels].reject { |label| existing_labels.include? label }
362 if issue.milestone.nil? or not missing_labels.empty?
363 puts "Updating milestone, labels on #{repo}##{issue.number}"
364 github.update_issue(repo, issue.number, options)
365 sleep THROTTLE_SECONDS
366 end
367 end
368 end
369
370
371 def make_deliverable_milestone(github, repo, deliverable, workpackage)
372 title = deliverable['label']
373
374 milestone = get_milestones(github, repo).find { |ms| ms.title == title }
375 if milestone.nil?
376 puts "Making milestone on #{repo}: #{title}"
377 body = DELIV_MILESTONE_TPL % {
378 wplabel: workpackage['label'],
379 wptitle: workpackage['title'],
380 title: deliverable['title'],
381 lead: deliverable['lead'],
382 date: deliverable['due_date'],
383 month: deliverable['month'],
384 nature: deliverable['nature'],
385 page: deliverable['page'],
386 }
387
388 milestone = github.create_milestone(repo, title,
389 :due_on => deliverable['due_date'],
390 :description => body,
391 )
392 # throttle creation calls to avoid flags for abuse
393 sleep THROTTLE_SECONDS
394 else
395 puts "Milestone #{repo}@#{title} exists"
396 end
397 return milestone.number
398 end
399
400
401 def populate_workpackage(github, repo, workpackage)
402 # populate issues for a given workpackage
403 label = workpackage['label']
404
405 readme_path = "#{label}/README.md"
406 begin
407 github.contents(repo, :path => readme_path)
408 rescue Octokit::NotFound
409 readme = README_TPL % {
410 title: "#{workpackage['label']}: #{workpackage['title']}",
411 lead: workpackage['lead'],
412 page: workpackage['page'],
413 }
414
415 puts "Creating readme at #{repo}/#{readme_path}"
416 puts readme
417 github.create_contents(repo, readme_path, "Creating #{readme_path}", readme)
418 # throttle creation calls to avoid flags for abuse
419 sleep THROTTLE_SECONDS
420 end
421
422 workpackage['tasks'].each_value do |task|
423 org_labels = [REVERSE_SITES[task['lead']]] + \
424 (task['partners'] or []).map {|site| REVERSE_SITES[site]}
425
426 make_task_issue(github, repo, task, workpackage,
427 :labels => [
428 'task',
429 workpackage['label'],
430 ] + org_labels
431 )
432 end
433 workpackage['deliverables'].each do |deliverable|
434 milestone = make_deliverable_milestone(github, repo, deliverable, workpackage)
435 make_deliverable_issue(github, repo, deliverable, workpackage, {
436 :milestone => milestone,
437 :labels => [
438 'deliverable',
439 workpackage['label'],
440 REVERSE_SITES[deliverable['lead']]
441 ]
442 })
443 end
444 end
445
446
447 def main(proposal_dir)
448 # run the whole thing
449
450 # verify that there's a GitHub token in .netrc
451 check_token
452
453 # create client
454 github = Octokit::Client.new(:netrc => true)
455 github.auto_paginate = true
456
457 load_pdata(proposal_dir).each do |wp|
458 populate_workpackage(github, REPO, wp)
459 end
460 end
461
462 if __FILE__ == $0
463 puts ARGV
464 if ARGV.length > 1
465 proposal_dir = ARGV[1]
466 else
467 proposal_dir = '.'
468 end
469 main(proposal_dir)
470 end