Debian Codegraph @

scripts/split_ref_by_bai_datasize.py

raw
1 #!/usr/bin/env python3
2 """
3 Create a list of regions by splitting a reference based on the amount of data in bam files.
4 Uses the `bai` index of the bam files. Useful for submitting jobs of equal size to a cluster.
5 """
6
7 import sys
8 import os
9 import argparse
10 import time
11 import logging
12
13 import struct
14 import numpy as np
15 from scipy import interpolate
16
17
18 DEFAULT_LOGGING_LEVEL = logging.INFO
19 MAX_LOGGING_LEVEL = logging.CRITICAL
20
21 def setup_logger(verbose_level):
22 fmt=('%(levelname)s %(asctime)s [%(module)s:%(lineno)s %(funcName)s] :: '
23 '%(message)s')
24 logging.basicConfig(format=fmt, level=max((0, min((MAX_LOGGING_LEVEL,
25 DEFAULT_LOGGING_LEVEL-(verbose_level*10))))))
26
27
28 def Main(argv):
29 tic_total = time.time()
30
31 # parse arguments
32 parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
33 parser.add_argument('bamfiles', metavar='BAMFILE', nargs='*')
34 parser.add_argument('-L', '--bam-list', nargs='*')
35 parser.add_argument('-r', '--reference-fai', help="reference fasta index file", required=True)
36 parser.add_argument('-s', '--target-data-size', default='100e6', help="target combined data size of bam files in each region (MB)")
37 parser.add_argument('--bai-interval-size', default=16384, type=int, help="Size in baseparis of each interval in the bam index (bai).")
38 parser.add_argument('-v', '--verbose', action='count', default=0,
39 help="increase logging verbosity")
40 parser.add_argument('-q', '--quiet', action='count', default=0,
41 help="decrease logging verbosity")
42 args = parser.parse_args(argv)
43
44 # setup logger
45 setup_logger(verbose_level=args.verbose-args.quiet)
46 if argv is not None:
47 logging.warning('Using passed arguments: '+str(argv))
48 logging.info('args: '+str(args))
49
50 # additional argument parsing and datatype handling
51 if not args.bamfiles and not args.bam_list:
52 logging.error("Must provide an BAMFILE and/or --bam-list argument")
53 sys.exit(2)
54 args.target_data_size = int(float(args.target_data_size))*1000000
55 logging.info('target-data-size: '+str(args.target_data_size)+' bytes')
56
57 # read bam-lists if provided
58 if args.bam_list:
59 for bamlistfile in args.bam_list:
60 with open(bamlistfile,'r') as fh:
61 for x in fh:
62 x = x.split('#')[0].strip()
63 if x:
64 args.bamfiles.append(x)
65 #logging.info('bam files: '+", ".join(args.bamfiles)) # output complete list of bam files being used
66
67 # read the reference fasta index
68 fai_chrom = []
69 fai_len = []
70 with open(args.reference_fai,'r') as fh:
71 for x in fh:
72 x = x.strip().split(sep='\t')
73 fai_chrom.append(str(x[0]))
74 fai_len.append(int(x[1]))
75
76
77 ## read bai indexes, skipping bin info
78 # list by chrom of number of intervals
79 n_intvs = np.array([int(np.ceil(x/args.bai_interval_size)) for x in fai_len])
80 # list by chrom of lists of interval offsets
81 icumsz = [] # cumulative size of data by interval
82 for i,n in enumerate(n_intvs):
83 icumsz.append(np.zeros((n,), dtype=np.int64))
84
85 for bamfn in args.bamfiles:
86 baifn = bamfn+'.bai'
87 with open(baifn,'rb') as fh:
88 logging.info("processing: "+baifn)
89 # filetype magic check
90 assert struct.unpack('4s', fh.read(4))[0] == b'BAI\x01'
91
92 # number of reference sequences (chroms)
93 n_ref = struct.unpack('i', fh.read(4))[0]
94 assert n_ref == len(fai_len), "fasta index and bam index have must have same number of chroms"
95
96 for ci in range(n_ref):
97 # skip over the binning index
98 n_bin = struct.unpack('i', fh.read(4))[0]
99 for bini in range(n_bin):
100 bin_id = struct.unpack('I', fh.read(4))[0]
101 n_chunk = struct.unpack('i', fh.read(4))[0]
102 fh.seek(n_chunk*16, os.SEEK_CUR)
103 # read interval index
104 n_intv = struct.unpack('i', fh.read(4))[0]
105 if n_intv > 0:
106 ioff = np.array(struct.unpack(str(n_intv)+'Q', fh.read(n_intv*8)), dtype=np.int64)
107 while( len(ioff) < len(icumsz[ci]) ):
108 ioff = np.append(ioff, ioff[-1]+1)
109 icumsz[ci] += ioff-ioff[0]
110
111
112
113 ## make the list of regions
114 regions = []
115
116 for ci,chrom in enumerate(fai_chrom):
117
118 # sanity check last point if there are more than one
119 if len(icumsz[ci]) > 1:
120 assert icumsz[ci][-1] >= icumsz[ci][-2]
121
122 # tiny chroms just get 1 region
123 if len(icumsz[ci]) < 2:
124 regions.extend([ (fai_chrom[ci], 0, fai_len[ci]) ])
125 continue
126 ds = icumsz[ci]
127 pos = np.arange(0, ds.shape[0])*args.bai_interval_size
128
129 # estimate total data size for the chrom
130 f = interpolate.interp1d(pos, ds, fill_value='extrapolate', kind='linear')
131 ds_total = f([fai_len[ci]])[0]
132
133 num_regions = int(np.ceil(ds_total/args.target_data_size))
134
135 # approx equal LENGTH regions
136 # tmp = np.linspace(0, fai_len[ci], num=num_regions+1, endpoint=True, dtype=int)
137
138 # approx equal DATA SIZE regions
139 f = interpolate.interp1d(ds, pos, fill_value='extrapolate', kind='linear')
140 dsx = np.linspace(0, ds_total, num=num_regions+1, endpoint=True, dtype=int)
141 tmp = f(dsx).astype(int)
142 # ensure we exactly hit the endpoints
143 tmp[0] = 0
144 tmp[-1] = fai_len[ci]
145
146 regions.extend([ (fai_chrom[ci], tmp[i], tmp[i+1]) for i in range(len(tmp)-1) ])
147
148 ## Output regions file
149 for r in regions:
150 print(*r, sep='\t')
151
152 logging.info("Number of chroms: {}".format(len(fai_len)))
153 logging.info("Number of splits: {}".format(len(regions)-len(fai_len)))
154 logging.info("Number of regions: {}".format(len(regions)))
155
156 logging.info("Done: {:.2f} sec elapsed".format(time.time()-tic_total))
157 return 0
158
159
160
161 #########################################################################
162 # Main loop hook... if run as script run main, else this is just a module
163 if __name__ == '__main__':
164 if 'TESTING_ARGS' in globals():
165 sys.exit(Main(argv=TESTING_ARGS))
166 else:
167 sys.exit(Main(argv=None))
168