Debian Codegraph @

applications/bed/sort-bed/src/update-sort-bed-migrate-candidates.py

raw ◀ ▶
Source reconstructed at patch state 0 of 1 — not the installed (fully patched) source.
1 Debian patch touch this file
1 #!/usr/bin/env python
2
3 #
4 # BEDOPS
5 # Copyright (C) 2011-2025 Shane Neph, Scott Kuehn and Alex Reynolds
6 #
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
11 #
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
16 #
17 # You should have received a copy of the GNU General Public License along
18 # with this program; if not, write to the Free Software Foundation, Inc.,
19 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20 #
21
22 from __future__ import absolute_import, division, print_function, unicode_literals
23
24 import sys
25 import os
26 import argparse
27 import errno
28 import subprocess
29 import json
30 import logging
31
32 name = "update-sort-bed-migrate-candidates"
33 citation = " citation: http://bioinformatics.oxfordjournals.org/content/28/14/1919.abstract"
34 authors = " authors: Alex Reynolds and Shane Neph"
35 version = " version: 2.4.42"
36 usage = """ $ update-sort-bed-migrate-candidates [ --dry-run ] [ --debug ]
37 [ --write-list |
38 --resort-immediately |
39 --resort-in-parallel-via-slurm
40 [ --slurm-memory <MB> ]
41 [ --slurm-partition <SLURM partition> ]
42 [ --slurm-workdir <working directory> ]
43 [ --slurm-output <SLURM output directory> ]
44 [ --slurm-error <SLURM error directory> ]
45 ]
46 [ --bedops-root-dir <bedops directory> ]
47 [ --bedextract-path <path> ]
48 [ --sort-bed-path <path> ]
49 [ --unstarch-path <path> ]
50 [ --starch-path <path> ]
51 [ --starchcat-path <path> ]
52 [ --update-sort-bed-slurm-path <path> ]
53 [ --update-sort-bed-starch-slurm-path <path> ]
54 --parent-dir <parent directory>
55 [ --non-recursive-search ]"""
56 help = """
57 The "update-sort-bed-migrate-candidates" utility recursively locates BED and
58 Starch files in the specified parent directory and tests if they require
59 re-sorting to conform to the updated, post-v2.4.20 "sort-bed" order.
60
61 Files with the extensions starch, bstarch, gstarch, bed, or bed[g|G]raph in
62 the parent directory are tested. Files without these extensions are ignored.
63
64 If the "--non-recursive-search" option is added, this utility will only search
65 within the specified parent directory, and go no deeper.
66
67 This utility offers one of three (exclusive) actions for migration:
68
69 1. Using "--write-list", files that require re-sorting can have their paths
70 written to standard output, which can be written to a file to be processed
71 later on, as desired.
72
73 2. Using "--resort-immediately", qualifying files can be resorted immediately
74 after all candidates are found, through a local, serial application of
75 'sort-bed'.
76
77 3. Using "--resort-in-parallel-via-slurm", candidate files can be migrated by
78 applying the 'update-sort-bed-slurm' script to resort in parallel on a
79 computational cluster managed with a SLURM job scheduler.
80
81 Note that one of these three options must be chosen, to perform the
82 stated action, and only one option can be selected.
83
84 When using "--resort-immediately" or "--resort-in-parallel-via-slurm", the
85 resorted files will have the name of the original BED or Starch file. The
86 original files will have their old name, with the ".backup" extension.
87
88 Use the "--bedops-root-dir" option to specify the directory containing the
89 BEDOPS toolkit binaries to be used for migration. You can provide more specific
90 paths to individual binaries, using the following options:
91
92 Add the "--bedextract-path", "--sort-bed-path", "--unstarch-path",
93 "--starch-path", "--starchcat-path", "--update-sort-bed-slurm", and/or
94 "--update-sort-bed-starch-slurm" options to specify custom paths to versions
95 of these tools, if desired. These values will be passed along to downstream
96 helper scripts that use them.
97
98 ---
99
100 Suggestions:
101
102 1. Add the "--dry-run" option to "--resort-immediately" or the
103 "--resort-in-parallel-via-slurm" options to see the behavior before any
104 filesystem actions are performed. Remove "--dry-run" to perform the
105 specified work.
106
107 2. If you use the resort-via-SLURM option, consider using "--slurm-memory",
108 "--slurm-partition", "--slurm-workdir", "--slurm-output", and
109 "--slurm-error" options to match the setup of your particular cluster
110 environment and inputs.
111
112 3. Add "--debug" option to log debug statements to get more detail about
113 internal operation of update process.
114
115 """
116
117 def main():
118 parser = argparse.ArgumentParser(prog=name, usage=usage, add_help=False)
119 parser.add_argument('--help', '-h', action='store_true', dest='help')
120 parser.add_argument('--write-list', '-l', action="store_true", dest='write_list')
121 parser.add_argument('--resort-immediately', '-i', action="store_true", dest='resort_immediately')
122 parser.add_argument('--resort-in-parallel-via-slurm', '-s', action="store_true", dest='resort_in_parallel_via_slurm')
123 parser.add_argument('--slurm-memory', '-m', type=str, action="store", dest='slurm_memory')
124 parser.add_argument('--slurm-partition', '-p', type=str, action="store", dest='slurm_partition')
125 parser.add_argument('--slurm-workdir', '-w', type=str, action="store", dest='slurm_workdir')
126 parser.add_argument('--slurm-output', '-r', type=str, action="store", dest='slurm_output')
127 parser.add_argument('--slurm-error', '-e', type=str, action="store", dest='slurm_error')
128 parser.add_argument('--bedops-root-dir', '-b', type=str, action="store", dest='bedops_root_dir')
129 parser.add_argument('--bedextract-path', '-x', type=str, action="store", dest='bedextract_path')
130 parser.add_argument('--sort-bed-path', '-o', type=str, action="store", dest='sort_bed_path')
131 parser.add_argument('--unstarch-path', '-u', type=str, action="store", dest='unstarch_path')
132 parser.add_argument('--starch-path', '-t', type=str, action="store", dest='starch_path')
133 parser.add_argument('--starchcat-path', '-z', type=str, action="store", dest='starchcat_path')
134 parser.add_argument('--update-sort-bed-slurm-path', '-y', type=str, action="store", dest='update_sort_bed_slurm_path')
135 parser.add_argument('--update-sort-bed-starch-slurm-path', '-q', type=str, action="store", dest='update_sort_bed_starch_slurm_path')
136 parser.add_argument('--dry-run', '-n', action="store_true", dest='dry_run')
137 parser.add_argument('--debug', '-d', action="store_true", dest='debug')
138 parser.add_argument('--parent-dir', '-a', type=str, action="store", dest='parent_dir')
139 parser.add_argument('--non-recursive-search', '-v', action="store_true", dest='non_recursive_search')
140 args = parser.parse_args()
141
142 action_counter = 0
143 if args.write_list:
144 action_counter += 1
145 if args.resort_immediately:
146 action_counter += 1
147 if args.resort_in_parallel_via_slurm:
148 action_counter += 1
149
150 if args.help or not args.parent_dir or action_counter != 1:
151 sys.stdout.write(name + '\n')
152 sys.stdout.write(citation + '\n')
153 sys.stdout.write(version + '\n')
154 sys.stdout.write(authors + '\n\n')
155 sys.stdout.write(usage + '\n')
156 sys.stdout.write(help)
157 if args.help:
158 sys.exit(os.EX_OK)
159 else:
160 sys.exit(errno.EINVAL)
161
162 logger = None
163 if args.debug:
164 logging.basicConfig(level=logging.DEBUG)
165 logger = logging.getLogger(__name__)
166
167 bedextract_path = None
168 if logger: logger.info('Locating \"bedextract\" binary')
169 if not args.bedextract_path and not args.bedops_root_dir:
170 bedextract_path = find_binary('bedextract')
171 if not bedextract_path:
172 if logger: logger.info('Could not locate \"bedextract\" binary')
173 sys.stderr.write("ERROR: This script must be run on a system with BEDOPS bedextract\n")
174 sys.exit(errno.EEXIST)
175 elif args.bedops_root_dir and not args.bedextract_path:
176 bedextract_path = os.path.join(args.bedops_root_dir, 'bedextract')
177 elif args.bedextract_path:
178 bedextract_path = args.bedextract_path
179 elif not cmd_exists('bedextract'):
180 if logger: logger.info('Could not locate \"bedextract\" binary')
181 sys.stderr.write("ERROR: This script must be run on a system with BEDOPS bedextract\n")
182 sys.exit(errno.EEXIST)
183 if logger: logger.info('Location of \"bedextract\" is set to [%s]' % (bedextract_path))
184
185 sort_bed_path = None
186 if logger: logger.info('Locating \"sort-bed\" binary')
187 if not args.sort_bed_path and not args.bedops_root_dir:
188 sort_bed_path = find_binary('sort-bed')
189 if not sort_bed_path:
190 if logger: logger.info('Could not locate \"sort-bed\" binary')
191 sys.stderr.write("ERROR: This script must be run on a system with BEDOPS sort-bed\n")
192 sys.exit(errno.EEXIST)
193 elif args.bedops_root_dir and not args.sort_bed_path:
194 sort_bed_path = os.path.join(args.bedops_root_dir, 'sort-bed')
195 elif args.sort_bed_path:
196 sort_bed_path = args.sort_bed_path
197 elif not cmd_exists('sort-bed'):
198 if logger: logger.info('Could not locate \"sort-bed\" binary')
199 sys.stderr.write("ERROR: This script must be run on a system with BEDOPS sort-bed\n")
200 sys.exit(errno.EEXIST)
201 if logger: logger.info('Location of \"sort-bed\" is set to [%s]' % (sort_bed_path))
202
203 unstarch_path = None
204 if logger: logger.info('Locating \"unstarch\" binary')
205 if not args.unstarch_path and not args.bedops_root_dir:
206 unstarch_path = find_binary('unstarch')
207 if not unstarch_path:
208 if logger: logger.info('Could not locate \"unstarch\" binary')
209 sys.stderr.write("ERROR: This script must be run on a system with BEDOPS unstarch\n")
210 sys.exit(errno.EEXIST)
211 elif args.bedops_root_dir and not args.unstarch_path:
212 unstarch_path = os.path.join(args.bedops_root_dir, 'unstarch')
213 elif args.unstarch_path:
214 unstarch_path = args.unstarch_path
215 elif not cmd_exists('unstarch'):
216 if logger: logger.info('Could not locate \"unstarch\" binary')
217 sys.stderr.write("ERROR: This script must be run on a system with BEDOPS unstarch\n")
218 sys.exit(errno.EEXIST)
219 if logger: logger.info('Location of \"unstarch\" is set to [%s]' % (unstarch_path))
220
221 starch_path = None
222 if logger: logger.info('Locating \"starch\" binary')
223 if not args.starch_path and not args.bedops_root_dir:
224 starch_path = find_binary('starch')
225 if not starch_path:
226 if logger: logger.info('Could not locate \"starch\" binary')
227 sys.stderr.write("ERROR: This script must be run on a system with BEDOPS starch\n")
228 sys.exit(errno.EEXIST)
229 elif args.bedops_root_dir and not args.starch_path:
230 starch_path = os.path.join(args.bedops_root_dir, 'starch')
231 elif args.starch_path:
232 starch_path = args.starch_path
233 elif not cmd_exists('starch'):
234 if logger: logger.info('Could not locate \"starch\" binary')
235 sys.stderr.write("ERROR: This script must be run on a system with BEDOPS starch\n")
236 sys.exit(errno.EEXIST)
237 if logger: logger.info('Location of \"starch\" is set to [%s]' % (starch_path))
238
239 starchcat_path = None
240 if logger: logger.info('Locating \"starchcat\" binary')
241 if not args.starchcat_path and not args.bedops_root_dir:
242 starchcat_path = find_binary('starchcat')
243 if not starchcat_path:
244 if logger: logger.info('Could not locate \"starchcat\" binary')
245 sys.stderr.write("ERROR: This script must be run on a system with BEDOPS starchcat\n")
246 sys.exit(errno.EEXIST)
247 elif args.bedops_root_dir and not args.starchcat_path:
248 starchcat_path = os.path.join(args.bedops_root_dir, 'starchcat')
249 elif args.starchcat_path:
250 starchcat_path = args.starchcat_path
251 elif not cmd_exists('starchcat'):
252 if logger: logger.info('Could not locate \"starchcat\" binary')
253 sys.stderr.write("ERROR: This script must be run on a system with BEDOPS starchcat\n")
254 sys.exit(errno.EEXIST)
255 if logger: logger.info('Location of \"starchcat\" is set to [%s]' % (starchcat_path))
256
257 update_sort_bed_slurm_path = None
258 if logger: logger.info('Locating \"update-sort-bed-slurm\" script')
259 if not args.update_sort_bed_slurm_path and not args.bedops_root_dir:
260 update_sort_bed_slurm_path = find_binary('update-sort-bed-slurm')
261 if not update_sort_bed_slurm_path:
262 if logger: logger.info('Could not locate \"update-sort-bed-slurm\" binary')
263 sys.stderr.write("ERROR: This script must be run on a system with BEDOPS update-sort-bed-slurm\n")
264 sys.exit(errno.EEXIST)
265 elif args.bedops_root_dir and not args.update_sort_bed_slurm_path:
266 update_sort_bed_slurm_path = os.path.join(args.bedops_root_dir, 'update-sort-bed-slurm')
267 elif args.update_sort_bed_slurm_path:
268 update_sort_bed_slurm_path = args.update_sort_bed_slurm_path
269 elif not cmd_exists('update-sort-bed-slurm'):
270 if logger: logger.info('Could not locate \"update-sort-bed-slurm\" binary')
271 sys.stderr.write("ERROR: This script must be run on a system with BEDOPS update-sort-bed-slurm\n")
272 sys.exit(errno.EEXIST)
273 if logger: logger.info('Location of \"update-sort-bed-slurm\" is set to [%s]' % (update_sort_bed_slurm_path))
274
275 update_sort_bed_starch_slurm_path = None
276 if logger: logger.info('Locating \"update-sort-bed-starch-slurm\" script')
277 if not args.update_sort_bed_starch_slurm_path or not args.bedops_root_dir:
278 update_sort_bed_starch_slurm_path = find_binary('update-sort-bed-starch-slurm')
279 if not update_sort_bed_starch_slurm_path:
280 if logger: logger.info('Could not locate \"update-sort-bed-starch-slurm\" binary')
281 sys.stderr.write("ERROR: This script must be run on a system with BEDOPS update-sort-bed-starch-slurm\n")
282 sys.exit(errno.EEXIST)
283 elif args.bedops_root_dir and not args.update_sort_bed_starch_slurm_path:
284 update_sort_bed_starch_slurm_path = os.path.join(args.bedops_root_dir, 'update-sort-bed-starch-slurm')
285 elif args.update_sort_bed_starch_slurm_path:
286 update_sort_bed_starch_slurm_path = args.update_sort_bed_starch_slurm_path
287 elif not cmd_exists('update-sort-bed-starch-slurm'):
288 if logger: logger.info('Could not locate \"update-sort-bed-starch-slurm\" binary')
289 sys.stderr.write("ERROR: This script must be run on a system with BEDOPS update-sort-bed-starch-slurm\n")
290 sys.exit(errno.EEXIST)
291 if logger: logger.info('Location of \"update-sort-bed-starch-slurm\" is set to [%s]' % (update_sort_bed_starch_slurm_path))
292
293 bed_candidates = []
294 starch_candidates = []
295 if logger: logger.info('Initializing unfiltered BED and Starch candidate lists')
296
297 for root, directories, filenames in os.walk(args.parent_dir):
298 for filename in filenames:
299 candidate = os.path.join(root, filename)
300 if candidate.lower().endswith(('.starch', 'bstarch', 'gstarch')):
301 starch_candidates.append(candidate)
302 if candidate.lower().endswith(('.bed', '.bedgraph', '.bedGraph')):
303 bed_candidates.append(candidate)
304 if args.non_recursive_search: break
305 if logger: logger.info('Unfiltered BED candidates are [%s]' % (str(bed_candidates)))
306 if logger: logger.info('Unfiltered Starch candidates are [%s]' % (str(starch_candidates)))
307
308 filtered_bed_candidates = []
309 filtered_starch_candidates = []
310 if logger: logger.info('Initializing filtered BED and Starch candidate lists')
311
312 if len(bed_candidates) > 0:
313 # run sort-bed --check-sort on each BED candidate
314 for candidate in bed_candidates:
315 test_bed_sort_cmd_components = [
316 sort_bed_path,
317 '--check-sort',
318 candidate,
319 '>',
320 '/dev/null'
321 ]
322 if logger: logger.info('Checking BED candidate via [%s]' % (' '.join(test_bed_sort_cmd_components)))
323 test_bed_sort_process = subprocess.Popen(' '.join(test_bed_sort_cmd_components),
324 shell=True,
325 stdin=subprocess.PIPE,
326 stdout=subprocess.PIPE,
327 stderr=subprocess.STDOUT,
328 close_fds=True)
329 (test_bed_sort_stdout, test_bed_sort_stderr) = test_bed_sort_process.communicate()
330 if test_bed_sort_process.returncode != 0:
331 filtered_bed_candidates.append(candidate)
332 if logger: logger.info('Filtered BED candidates are [%s]' % (str(filtered_bed_candidates)))
333
334 if len(starch_candidates) > 0:
335 # run sort-bed --check-sort on the extracted output of a Starch candidate
336 for candidate in starch_candidates:
337 get_archive_version_cmd_components = [
338 unstarch_path,
339 '--list-json',
340 candidate
341 ]
342 try:
343 get_archive_version_cmd_result = subprocess.check_output(get_archive_version_cmd_components)
344 except subprocess.CalledProcessError as err:
345 get_archive_version_cmd_result = "ERROR: Command '{}' returned with error (code {}): {}".format(err.cmd, err.returncode, err.output)
346 raise
347 archive_metadata = json.loads(get_archive_version_cmd_result.decode('utf-8'))
348 try:
349 archive_version = archive_metadata['archive']['version']
350 except KeyError as err:
351 sys.stderr.write("ERROR: Could not read archive version from Starch metadata\n")
352 raise
353
354 if archive_version['major'] < 2 or (archive_version['major'] == 2 and archive_version['minor'] < 2):
355 test_starch_sort_cmd_components = [
356 unstarch_path,
357 candidate,
358 '|',
359 sort_bed_path,
360 '--check-sort',
361 '-',
362 '>',
363 '/dev/null'
364 ]
365 if logger: logger.info('Checking Starch candidate via [%s]' % (' '.join(test_starch_sort_cmd_components)))
366 test_starch_sort_process = subprocess.Popen(' '.join(test_starch_sort_cmd_components),
367 shell=True,
368 stdin=subprocess.PIPE,
369 stdout=subprocess.PIPE,
370 stderr=subprocess.STDOUT,
371 close_fds=True)
372 (test_starch_sort_stdout, test_starch_sort_stderr) = test_starch_sort_process.communicate()
373 if test_starch_sort_process.returncode != 0:
374 filtered_starch_candidates.append(candidate)
375 if logger: logger.info('Filtered Starch candidates are [%s]' % (str(filtered_starch_candidates)))
376
377 all_candidates = merge(filtered_bed_candidates, filtered_starch_candidates)
378 if logger: logger.info('Merged filtered BED and Starch candidate lists into all-candidate list')
379
380 if args.write_list:
381 if logger: logger.info('Writing candidate paths to standard output stream')
382 for candidate in all_candidates:
383 sys.stdout.write("%s\n" % (candidate))
384
385 if args.resort_immediately:
386 if logger: logger.info('Resorting candidates immediately')
387 for candidate in all_candidates:
388 #
389 # Filenames
390 #
391 src_original_fn = candidate
392 src_backup_fn = src_original_fn + '.backup'
393 dest_new_fn = src_original_fn + '.new'
394 dest_final_fn = src_original_fn
395 #
396 # BED
397 #
398 if candidate.lower().endswith(('.bed', '.bedgraph', '.bedGraph')):
399 if logger: logger.info('Planning to resort BED candidate [%s]' % (src_original_fn))
400 bed_resort_cmd_components = [
401 sort_bed_path,
402 src_original_fn,
403 '>',
404 dest_new_fn
405 ]
406 if logger: logger.info('Planning to resort BED candidate via [%s]' % (' '.join(bed_resort_cmd_components)))
407 if not args.dry_run:
408 bed_resort_process = subprocess.Popen(' '.join(bed_resort_cmd_components),
409 shell=True,
410 stdin=subprocess.PIPE,
411 stdout=subprocess.PIPE,
412 stderr=subprocess.STDOUT,
413 close_fds=True)
414 (bed_resort_stdout, bed_resort_stderr) = bed_resort_process.communicate()
415 if bed_resort_process.returncode != 0:
416 raise
417 else:
418 if logger: logger.info('Planning to rename [%s] to [%s]' % (src_original_fn, src_backup_fn))
419 try:
420 os.rename(src_original_fn, src_backup_fn)
421 if logger: logger.info('Renamed [%s] to [%s]' % (src_original_fn, src_backup_fn))
422 except OSError as err:
423 sys.stderr.write("Error: Could not rename [%s] to [%s]\n" % (src_original_fn, src_backup_fn))
424 raise
425 if logger: logger.info('Planning to rename [%s] to [%s]' % (dest_new_fn, dest_final_fn))
426 try:
427 os.rename(dest_new_fn, dest_final_fn)
428 if logger: logger.info('Renamed [%s] to [%s]' % (dest_new_fn, dest_final_fn))
429 except OSError as err:
430 sys.stderr.write("Error: Could not rename [%s] to [%s]\n" % (dest_new_fn, dest_final_fn))
431 raise
432 if logger: logger.info('Resorted BED candidate [%s] to [%s]' % (src_original_fn, dest_final_fn))
433 #
434 # Starch
435 #
436 if candidate.lower().endswith(('.starch', 'bstarch', 'gstarch')):
437 if logger: logger.info('Planning to resort Starch candidate [%s]' % (src_original_fn))
438 starch_resort_cmd_components = [
439 unstarch_path,
440 src_original_fn,
441 '|',
442 sort_bed_path,
443 '-',
444 '|',
445 starch_path,
446 '-',
447 '>',
448 dest_new_fn
449 ]
450 if logger: logger.info('Planning to resort Starch candidate via [%s]' % (' '.join(starch_resort_cmd_components)))
451 if not args.dry_run:
452 starch_resort_process = subprocess.Popen(' '.join(starch_resort_cmd_components),
453 shell=True,
454 stdin=subprocess.PIPE,
455 stdout=subprocess.PIPE,
456 stderr=subprocess.STDOUT,
457 close_fds=True)
458 (starch_resort_stdout, starch_resort_stderr) = starch_resort_process.communicate()
459 if starch_resort_process.returncode != 0:
460 raise
461 else:
462 if logger: logger.info('Debug: Planning to rename [%s] to [%s]' % (src_original_fn, src_backup_fn))
463 try:
464 os.rename(src_original_fn, src_backup_fn)
465 if logger: logger.info('Renamed [%s] to [%s]' % (src_original_fn, src_backup_fn))
466 except OSError as err:
467 sys.stderr.write("Error: Could not rename [%s] to [%s]\n" % (src_original_fn, src_backup_fn))
468 raise
469 sys.stderr.write("Debug: Planning to rename [%s] to [%s]\n" % (dest_new_fn, dest_final_fn))
470 try:
471 os.rename(dest_new_fn, dest_final_fn)
472 if logger: logger.info('Renamed [%s] to [%s]' % (dest_new_fn, dest_final_fn))
473 except OSError as err:
474 sys.stderr.write("Error: Could not rename [%s] to [%s]\n" % (dest_new_fn, dest_final_fn))
475 raise
476 if logger: logger.info('Resorted Starch candidate [%s] to [%s]' % (src_original_fn, dest_final_fn))
477
478 if args.resort_in_parallel_via_slurm:
479 if logger: logger.info('Resorting candidates in parallel via SLURM job scheduler')
480 for candidate in all_candidates:
481 #
482 # Filenames
483 #
484 src_original_fn = candidate
485 src_backup_fn = src_original_fn + '.backup'
486 dest_temp_fn = src_original_fn + '.temp'
487 dest_final_fn = src_original_fn
488 #
489 # BED
490 #
491 if candidate.lower().endswith(('.bed', '.bedgraph', '.bedGraph')):
492 if logger: logger.info('Planning to resort BED candidate [%s]' % (src_original_fn))
493 bed_slurm_resort_cmd_components = [
494 update_sort_bed_slurm_path,
495 '--input-original',
496 src_original_fn,
497 '--input-backup',
498 src_backup_fn,
499 '--output-temp',
500 dest_temp_fn,
501 '--output-final',
502 dest_final_fn,
503 '--bedextract-path',
504 bedextract_path,
505 '--sort-bed-path',
506 sort_bed_path
507 ]
508 slurm_options = customize_slurm_options(args)
509 if slurm_options:
510 bed_slurm_resort_cmd_components.append(slurm_options)
511 if args.debug:
512 bed_slurm_resort_cmd_components.append('--debug')
513 if logger: logger.info('Planning to resort BED candidate via [%s]' % (' '.join(bed_slurm_resort_cmd_components)))
514 if not args.dry_run:
515 bed_slurm_resort_process = subprocess.Popen(' '.join(bed_slurm_resort_cmd_components),
516 shell=True,
517 stdin=subprocess.PIPE,
518 stdout=subprocess.PIPE,
519 stderr=subprocess.STDOUT,
520 close_fds=True)
521 (bed_slurm_resort_stdout, bed_slurm_resort_stderr) = bed_slurm_resort_process.communicate()
522 if bed_slurm_resort_process.returncode != 0:
523 raise
524 if logger: logger.info('SLURM task submitted to resort [%s]' % (src_original_fn))
525
526 # Starch
527 if candidate.lower().endswith(('.starch', 'bstarch', 'gstarch')):
528 if logger: logger.info('Planning to resort Starch candidate [%s]' % (src_original_fn))
529 starch_slurm_resort_cmd_components = [
530 update_sort_bed_starch_slurm_path,
531 '--input-original',
532 src_original_fn,
533 '--input-backup',
534 src_backup_fn,
535 '--output-temp',
536 dest_temp_fn,
537 '--output-final',
538 dest_final_fn,
539 '--bedextract-path',
540 bedextract_path,
541 '--sort-bed-path',
542 sort_bed_path,
543 '--unstarch-path',
544 unstarch_path,
545 '--starch-path',
546 starch_path,
547 '--starchcat-path',
548 starchcat_path
549 ]
550 slurm_options = customize_slurm_options(args)
551 if slurm_options:
552 starch_slurm_resort_cmd_components.append(slurm_options)
553 if args.debug:
554 starch_slurm_resort_cmd_components.append('--debug')
555 if logger: logger.info('Planning to resort Starch candidate via [%s]' % (' '.join(starch_slurm_resort_cmd_components)))
556 if not args.dry_run:
557 starch_slurm_resort_process = subprocess.Popen(' '.join(starch_slurm_resort_cmd_components),
558 shell=True,
559 stdin=subprocess.PIPE,
560 stdout=subprocess.PIPE,
561 stderr=subprocess.STDOUT,
562 close_fds=True)
563 (starch_slurm_resort_stdout, starch_slurm_resort_stderr) = starch_slurm_resort_process.communicate()
564 if starch_slurm_resort_process.returncode != 0:
565 raise
566 if logger: logger.info('SLURM task submitted to resort [%s]' % (src_fn))
567
568 def find_binary(binary_to_find):
569 for p in os.environ['PATH'].split(':'):
570 for r, d, f in os.walk(p):
571 for filename in f:
572 if filename == binary_to_find:
573 return os.path.join(r, filename)
574 return None
575
576 def cmd_exists(cmd):
577 return subprocess.call("type " + cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0
578
579 def customize_slurm_options(args):
580 custom_slurm_options = ""
581 if args.slurm_memory:
582 custom_slurm_options += "--slurm-memory " + args.slurm_memory
583 if args.slurm_partition:
584 custom_slurm_options += "--slurm-partition " + args.slurm_partition
585 if args.slurm_workdir:
586 custom_slurm_options += "--slurm-workdir " + args.slurm_workdir
587 if args.slurm_output:
588 custom_slurm_options += "--slurm-output " + args.slurm_output
589 if args.slurm_error:
590 custom_slurm_options += "--slurm-error " + args.slurm_error
591 return custom_slurm_options if len(custom_slurm_options) > 0 else None
592
593 def merge(l, m):
594 result = []
595 i = j = 0
596 total = len(l) + len(m)
597 while len(result) != total:
598 if len(l) == i:
599 result += m[j:]
600 break
601 elif len(m) == j:
602 result += l[i:]
603 break
604 elif l[i] < m[j]:
605 result.append(l[i])
606 i += 1
607 else:
608 result.append(m[j])
609 j += 1
610 return result
611
612 if __name__ == "__main__":
613 main()