| 1 |
#!/usr/bin/env python3 |
| 2 |
""" |
| 3 |
Create a list of regions by splitting a reference based on the amount of data in bam files. |
| 4 |
Uses the `bai` index of the bam files. Useful for submitting jobs of equal size to a cluster. |
| 5 |
""" |
| 6 |
|
| 7 |
import sys |
| 8 |
import os |
| 9 |
import argparse |
| 10 |
import time |
| 11 |
import logging |
| 12 |
|
| 13 |
import struct |
| 14 |
import numpy as np |
| 15 |
from scipy import interpolate |
| 16 |
|
| 17 |
|
| 18 |
DEFAULT_LOGGING_LEVEL = logging.INFO |
| 19 |
MAX_LOGGING_LEVEL = logging.CRITICAL |
| 20 |
|
| 21 |
def setup_logger(verbose_level): |
| 22 |
fmt=('%(levelname)s %(asctime)s [%(module)s:%(lineno)s %(funcName)s] :: ' |
| 23 |
'%(message)s') |
| 24 |
logging.basicConfig(format=fmt, level=max((0, min((MAX_LOGGING_LEVEL, |
| 25 |
DEFAULT_LOGGING_LEVEL-(verbose_level*10)))))) |
| 26 |
|
| 27 |
|
| 28 |
def Main(argv): |
| 29 |
tic_total = time.time() |
| 30 |
|
| 31 |
# parse arguments |
| 32 |
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) |
| 33 |
parser.add_argument('bamfiles', metavar='BAMFILE', nargs='*') |
| 34 |
parser.add_argument('-L', '--bam-list', nargs='*') |
| 35 |
parser.add_argument('-r', '--reference-fai', help="reference fasta index file", required=True) |
| 36 |
parser.add_argument('-s', '--target-data-size', default='100e6', help="target combined data size of bam files in each region (MB)") |
| 37 |
parser.add_argument('--bai-interval-size', default=16384, type=int, help="Size in baseparis of each interval in the bam index (bai).") |
| 38 |
parser.add_argument('-v', '--verbose', action='count', default=0, |
| 39 |
help="increase logging verbosity") |
| 40 |
parser.add_argument('-q', '--quiet', action='count', default=0, |
| 41 |
help="decrease logging verbosity") |
| 42 |
args = parser.parse_args(argv) |
| 43 |
|
| 44 |
# setup logger |
| 45 |
setup_logger(verbose_level=args.verbose-args.quiet) |
| 46 |
if argv is not None: |
| 47 |
logging.warning('Using passed arguments: '+str(argv)) |
| 48 |
logging.info('args: '+str(args)) |
| 49 |
|
| 50 |
# additional argument parsing and datatype handling |
| 51 |
if not args.bamfiles and not args.bam_list: |
| 52 |
logging.error("Must provide an BAMFILE and/or --bam-list argument") |
| 53 |
sys.exit(2) |
| 54 |
args.target_data_size = int(float(args.target_data_size))*1000000 |
| 55 |
logging.info('target-data-size: '+str(args.target_data_size)+' bytes') |
| 56 |
|
| 57 |
# read bam-lists if provided |
| 58 |
if args.bam_list: |
| 59 |
for bamlistfile in args.bam_list: |
| 60 |
with open(bamlistfile,'r') as fh: |
| 61 |
for x in fh: |
| 62 |
x = x.split('#')[0].strip() |
| 63 |
if x: |
| 64 |
args.bamfiles.append(x) |
| 65 |
#logging.info('bam files: '+", ".join(args.bamfiles)) # output complete list of bam files being used |
| 66 |
|
| 67 |
# read the reference fasta index |
| 68 |
fai_chrom = [] |
| 69 |
fai_len = [] |
| 70 |
with open(args.reference_fai,'r') as fh: |
| 71 |
for x in fh: |
| 72 |
x = x.strip().split(sep='\t') |
| 73 |
fai_chrom.append(str(x[0])) |
| 74 |
fai_len.append(int(x[1])) |
| 75 |
|
| 76 |
|
| 77 |
## read bai indexes, skipping bin info |
| 78 |
# list by chrom of number of intervals |
| 79 |
n_intvs = np.array([int(np.ceil(x/args.bai_interval_size)) for x in fai_len]) |
| 80 |
# list by chrom of lists of interval offsets |
| 81 |
icumsz = [] # cumulative size of data by interval |
| 82 |
for i,n in enumerate(n_intvs): |
| 83 |
icumsz.append(np.zeros((n,), dtype=np.int64)) |
| 84 |
|
| 85 |
for bamfn in args.bamfiles: |
| 86 |
baifn = bamfn+'.bai' |
| 87 |
with open(baifn,'rb') as fh: |
| 88 |
logging.info("processing: "+baifn) |
| 89 |
# filetype magic check |
| 90 |
assert struct.unpack('4s', fh.read(4))[0] == b'BAI\x01' |
| 91 |
|
| 92 |
# number of reference sequences (chroms) |
| 93 |
n_ref = struct.unpack('i', fh.read(4))[0] |
| 94 |
assert n_ref == len(fai_len), "fasta index and bam index have must have same number of chroms" |
| 95 |
|
| 96 |
for ci in range(n_ref): |
| 97 |
# skip over the binning index |
| 98 |
n_bin = struct.unpack('i', fh.read(4))[0] |
| 99 |
for bini in range(n_bin): |
| 100 |
bin_id = struct.unpack('I', fh.read(4))[0] |
| 101 |
n_chunk = struct.unpack('i', fh.read(4))[0] |
| 102 |
fh.seek(n_chunk*16, os.SEEK_CUR) |
| 103 |
# read interval index |
| 104 |
n_intv = struct.unpack('i', fh.read(4))[0] |
| 105 |
if n_intv > 0: |
| 106 |
ioff = np.array(struct.unpack(str(n_intv)+'Q', fh.read(n_intv*8)), dtype=np.int64) |
| 107 |
while( len(ioff) < len(icumsz[ci]) ): |
| 108 |
ioff = np.append(ioff, ioff[-1]+1) |
| 109 |
icumsz[ci] += ioff-ioff[0] |
| 110 |
|
| 111 |
|
| 112 |
|
| 113 |
## make the list of regions |
| 114 |
regions = [] |
| 115 |
|
| 116 |
for ci,chrom in enumerate(fai_chrom): |
| 117 |
|
| 118 |
# sanity check last point if there are more than one |
| 119 |
if len(icumsz[ci]) > 1: |
| 120 |
assert icumsz[ci][-1] >= icumsz[ci][-2] |
| 121 |
|
| 122 |
# tiny chroms just get 1 region |
| 123 |
if len(icumsz[ci]) < 2: |
| 124 |
regions.extend([ (fai_chrom[ci], 0, fai_len[ci]) ]) |
| 125 |
continue |
| 126 |
ds = icumsz[ci] |
| 127 |
pos = np.arange(0, ds.shape[0])*args.bai_interval_size |
| 128 |
|
| 129 |
# estimate total data size for the chrom |
| 130 |
f = interpolate.interp1d(pos, ds, fill_value='extrapolate', kind='linear') |
| 131 |
ds_total = f([fai_len[ci]])[0] |
| 132 |
|
| 133 |
num_regions = int(np.ceil(ds_total/args.target_data_size)) |
| 134 |
|
| 135 |
# approx equal LENGTH regions |
| 136 |
# tmp = np.linspace(0, fai_len[ci], num=num_regions+1, endpoint=True, dtype=int) |
| 137 |
|
| 138 |
# approx equal DATA SIZE regions |
| 139 |
f = interpolate.interp1d(ds, pos, fill_value='extrapolate', kind='linear') |
| 140 |
dsx = np.linspace(0, ds_total, num=num_regions+1, endpoint=True, dtype=int) |
| 141 |
tmp = f(dsx).astype(int) |
| 142 |
# ensure we exactly hit the endpoints |
| 143 |
tmp[0] = 0 |
| 144 |
tmp[-1] = fai_len[ci] |
| 145 |
|
| 146 |
regions.extend([ (fai_chrom[ci], tmp[i], tmp[i+1]) for i in range(len(tmp)-1) ]) |
| 147 |
|
| 148 |
## Output regions file |
| 149 |
for r in regions: |
| 150 |
print(*r, sep='\t') |
| 151 |
|
| 152 |
logging.info("Number of chroms: {}".format(len(fai_len))) |
| 153 |
logging.info("Number of splits: {}".format(len(regions)-len(fai_len))) |
| 154 |
logging.info("Number of regions: {}".format(len(regions))) |
| 155 |
|
| 156 |
logging.info("Done: {:.2f} sec elapsed".format(time.time()-tic_total)) |
| 157 |
return 0 |
| 158 |
|
| 159 |
|
| 160 |
|
| 161 |
######################################################################### |
| 162 |
# Main loop hook... if run as script run main, else this is just a module |
| 163 |
if __name__ == '__main__': |
| 164 |
if 'TESTING_ARGS' in globals(): |
| 165 |
sys.exit(Main(argv=TESTING_ARGS)) |
| 166 |
else: |
| 167 |
sys.exit(Main(argv=None)) |
| 168 |
|