// This file is part of PLINK 1.9, copyright (C) 2005-2024 Shaun Purcell,
// Christopher Chang.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see .
#include "plink_common.h"
// #include "pigz.h"
// no leading \n since this is used in LOGPRINTFWW expressions
const char g_errstr_fopen[] = "Error: Failed to open %s.\n";
const char g_cmdline_format_str[] = "\n " PROG_NAME_STR " [command flag(s)...] [other flag(s)...]\n " PROG_NAME_STR " --help [flag name(s)...]\n\n";
char g_textbuf[TEXTBUF_SIZE];
// note that \xxx character constants are interpreted in octal.
// technically no need to represent 0-31, but 64 extra bytes of data is
// probably cheaper than the code to subtract 32 everywhere.
const char g_one_char_strs[] = "\0\0\1\0\2\0\3\0\4\0\5\0\6\0\7\0\10\0\11\0\12\0\13\0\14\0\15\0\16\0\17\0\20\0\21\0\22\0\23\0\24\0\25\0\26\0\27\0\30\0\31\0\32\0\33\0\34\0\35\0\36\0\37\0\40\0\41\0\42\0\43\0\44\0\45\0\46\0\47\0\50\0\51\0\52\0\53\0\54\0\55\0\56\0\57\0\60\0\61\0\62\0\63\0\64\0\65\0\66\0\67\0\70\0\71\0\72\0\73\0\74\0\75\0\76\0\77\0\100\0\101\0\102\0\103\0\104\0\105\0\106\0\107\0\110\0\111\0\112\0\113\0\114\0\115\0\116\0\117\0\120\0\121\0\122\0\123\0\124\0\125\0\126\0\127\0\130\0\131\0\132\0\133\0\134\0\135\0\136\0\137\0\140\0\141\0\142\0\143\0\144\0\145\0\146\0\147\0\150\0\151\0\152\0\153\0\154\0\155\0\156\0\157\0\160\0\161\0\162\0\163\0\164\0\165\0\166\0\167\0\170\0\171\0\172\0\173\0\174\0\175\0\176\0\177\0\200\0\201\0\202\0\203\0\204\0\205\0\206\0\207\0\210\0\211\0\212\0\213\0\214\0\215\0\216\0\217\0\220\0\221\0\222\0\223\0\224\0\225\0\226\0\227\0\230\0\231\0\232\0\233\0\234\0\235\0\236\0\237\0\240\0\241\0\242\0\243\0\244\0\245\0\246\0\247\0\250\0\251\0\252\0\253\0\254\0\255\0\256\0\257\0\260\0\261\0\262\0\263\0\264\0\265\0\266\0\267\0\270\0\271\0\272\0\273\0\274\0\275\0\276\0\277\0\300\0\301\0\302\0\303\0\304\0\305\0\306\0\307\0\310\0\311\0\312\0\313\0\314\0\315\0\316\0\317\0\320\0\321\0\322\0\323\0\324\0\325\0\326\0\327\0\330\0\331\0\332\0\333\0\334\0\335\0\336\0\337\0\340\0\341\0\342\0\343\0\344\0\345\0\346\0\347\0\350\0\351\0\352\0\353\0\354\0\355\0\356\0\357\0\360\0\361\0\362\0\363\0\364\0\365\0\366\0\367\0\370\0\371\0\372\0\373\0\374\0\375\0\376\0\377";
const char* g_missing_geno_ptr = &(g_one_char_strs[96]);
const char* g_output_missing_geno_ptr = &(g_one_char_strs[96]);
char g_missing_geno_char2 = '.';
uintptr_t g_failed_alloc_attempt_size = 0;
sfmt_t g_sfmt;
FILE* g_logfile = nullptr;
char g_logbuf[LOGBUFLEN];
uint32_t g_debug_on = 0;
uint32_t g_log_failed = 0;
uint32_t g_thread_ct;
uint32_t aligned_malloc(uintptr_t size, uintptr_t** aligned_pp) {
#if defined __LP64__ && !defined __APPLE__
// Avoid random segfaults on 64-bit machines which have 8-byte- instead of
// 16-byte-aligned malloc(). (Slightly different code is needed if malloc()
// does not even guarantee 8-byte alignment.)
uintptr_t* malloc_ptr = (uintptr_t*)malloc(size + VEC_BYTES);
if (!malloc_ptr) {
g_failed_alloc_attempt_size = size + VEC_BYTES;
return 1;
}
*aligned_pp = (uintptr_t*)((((uintptr_t)malloc_ptr) + VEC_BYTES) & (~(VEC_BYTES_M1 * ONELU)));
(*aligned_pp)[-1] = (uintptr_t)malloc_ptr;
#else
// no SSE2 concerns here
*aligned_pp = (uintptr_t*)malloc(size);
if (!(*aligned_pp)) {
g_failed_alloc_attempt_size = size;
return 1;
}
#endif
return 0;
}
void aligned_free(uintptr_t* aligned_pp) {
#if defined __LP64__ && !defined __APPLE__
free((uintptr_t*)(aligned_pp[-1]));
#else
free(aligned_pp);
#endif
}
uint32_t push_ll_str(const char* ss, Ll_str** ll_stack_ptr) {
uintptr_t str_bytes = strlen(ss) + 1;
Ll_str* new_ll_str = (Ll_str*)malloc(sizeof(Ll_str) + str_bytes);
if (!new_ll_str) {
g_failed_alloc_attempt_size = sizeof(Ll_str) + str_bytes;
return 1;
}
new_ll_str->next = *ll_stack_ptr;
memcpy(new_ll_str->ss, ss, str_bytes);
*ll_stack_ptr = new_ll_str;
return 0;
}
void logstr(const char* ss) {
if (!g_debug_on) {
fputs(ss, g_logfile);
if (ferror(g_logfile)) {
putc_unlocked('\n', stdout);
fflush(stdout);
fprintf(stderr, "Warning: Logging failure on:\n%s\nFurther logging will not be attempted in this run.\n", ss);
g_log_failed = 1;
}
} else {
if (g_log_failed) {
fflush(stdout);
fputs(ss, stderr);
} else {
fputs(ss, g_logfile);
if (ferror(g_logfile)) {
putc_unlocked('\n', stdout);
fflush(stdout);
fprintf(stderr, "Error: Debug logging failure. Dumping to stderr:\n%s", ss);
g_log_failed = 1;
} else {
fflush(g_logfile);
}
}
}
}
void logprint(const char* ss) {
logstr(ss);
fputs(ss, stdout);
}
void logerrprint(const char* ss) {
logstr(ss);
fflush(stdout);
fputs(ss, stderr);
}
void logprintb() {
logstr(g_logbuf);
fputs(g_logbuf, stdout);
}
void logerrprintb() {
logstr(g_logbuf);
fflush(stdout);
fputs(g_logbuf, stderr);
}
void wordwrap(uint32_t suffix_len, char* ss) {
// Input: A null-terminated string with no intermediate newlines. If
// suffix_len is zero, there should be a terminating \n; otherwise,
// the last character should be a space.
// Effect: Spaces are replaced with newlines in a manner that plays well with
// 80 column terminal windows. (Multi-space blocks are never
// collapsed.)
char* token_start = ss;
char* line_end = &(ss[79]);
char* token_end;
while (1) {
while (*token_start == ' ') {
token_start++;
}
if (token_start > line_end) {
do {
*line_end = '\n';
line_end = &(line_end[80]);
} while (token_start > line_end);
}
token_end = strchr(token_start, ' ');
if (!token_end) {
if (&(token_start[79]) == line_end) {
return;
}
token_end = strchr(token_start, '\0');
if (!suffix_len) {
if (token_end <= &(line_end[1])) {
// okay if end-of-string is one past the end, because function
// assumes last character is \n in suffix_len == 0 case
assert(token_end[-1] == '\n');
return;
}
} else {
if (&(token_end[suffix_len]) <= line_end) {
return;
}
// because of terminal space assumption, token_start actually points
// to the end of the string
assert(token_start[-1] == ' ');
}
token_start[-1] = '\n';
return;
}
if (token_end > line_end) {
if (&(token_start[79]) != line_end) {
token_start[-1] = '\n';
line_end = &(token_start[79]);
if (token_end > line_end) {
// single really long token, can't do anything beyond putting it on
// its own line
*token_end = '\n';
line_end = &(token_end[80]);
}
} else {
// single really long token, *and* previous token was either
// nonexistent or long
*token_end = '\n';
line_end = &(token_end[80]);
}
}
token_start = &(token_end[1]);
}
}
void wordwrapb(uint32_t suffix_len) {
wordwrap(suffix_len, g_logbuf);
}
int32_t fopen_checked(const char* fname, const char* mode, FILE** target_ptr) {
*target_ptr = fopen(fname, mode);
if (!(*target_ptr)) {
LOGERRPRINTFWW(g_errstr_fopen, fname);
return -1;
}
return 0;
}
int32_t fwrite_checked(const void* buf, size_t len, FILE* outfile) {
while (len > 0x7ffff000) {
// OS X can't perform 2GB+ writes
// typical disk block size is 4kb, so 0x7ffff000 is the largest sensible
// write size
fwrite(buf, 1, 0x7ffff000, outfile);
buf = &(((unsigned char*)buf)[0x7ffff000]);
len -= 0x7ffff000;
}
fwrite(buf, 1, len, outfile);
return ferror(outfile);
}
int32_t gzopen_read_checked(const char* fname, gzFile* gzf_ptr) {
*gzf_ptr = gzopen(fname, FOPEN_RB);
if (!(*gzf_ptr)) {
LOGERRPRINTFWW(g_errstr_fopen, fname);
return RET_OPEN_FAIL;
}
if (gzbuffer(*gzf_ptr, 131072)) {
return RET_NOMEM;
}
return 0;
}
// manually managed, very large stack
unsigned char* g_bigstack_base;
unsigned char* g_bigstack_end;
unsigned char* bigstack_alloc(uintptr_t size) {
unsigned char* alloc_ptr;
size = round_up_pow2(size, CACHELINE);
if (bigstack_left() < size) {
g_failed_alloc_attempt_size = size;
return nullptr;
}
alloc_ptr = g_bigstack_base;
g_bigstack_base += size;
return alloc_ptr;
}
void bigstack_shrink_top(const void* rebase, uintptr_t new_size) {
uintptr_t freed_bytes = ((uintptr_t)(g_bigstack_base - ((unsigned char*)rebase))) - round_up_pow2(new_size, CACHELINE);
g_bigstack_base -= freed_bytes;
}
unsigned char* bigstack_end_alloc_presized(uintptr_t size) {
assert(!(size & END_ALLOC_CHUNK_M1));
uintptr_t cur_bigstack_left = bigstack_left();
if (size > cur_bigstack_left) {
g_failed_alloc_attempt_size = size;
return nullptr;
} else {
g_bigstack_end -= size;
return g_bigstack_end;
}
}
uint32_t match_upper(const char* ss, const char* fixed_str) {
char cc = *fixed_str++;
do {
if ((((unsigned char)(*ss++)) & 0xdf) != ((unsigned char)cc)) {
return 0;
}
cc = *fixed_str++;
} while (cc);
return !(*ss);
}
uint32_t match_upper_counted(const char* ss, const char* fixed_str, uint32_t ct) {
do {
if ((((unsigned char)(*ss++)) & 0xdf) != ((unsigned char)(*fixed_str++))) {
return 0;
}
} while (--ct);
return 1;
}
#ifdef __LP64__
static inline uint32_t scan_uint_capped_finish(const char* ss, uint64_t cap, uint32_t* valp) {
uint64_t val = *valp;
while (1) {
// a little bit of unrolling seems to help
const uint64_t cur_digit = (uint64_t)((unsigned char)(*ss++)) - 48;
if (cur_digit >= 10) {
break;
}
// val = val * 10 + cur_digit;
const uint64_t cur_digit2 = (uint64_t)((unsigned char)(*ss++)) - 48;
if (cur_digit2 >= 10) {
val = val * 10 + cur_digit;
if (val > cap) {
return 1;
}
break;
}
val = val * 100 + cur_digit * 10 + cur_digit2;
if (val > cap) {
return 1;
}
}
*valp = val;
return 0;
}
uint32_t scan_posint_capped(const char* ss, uint64_t cap, uint32_t* valp) {
// '0' has ascii code 48
*valp = (uint32_t)((unsigned char)(*ss++)) - 48;
if (*valp >= 10) {
// permit leading '+' (ascii 43), but not '++' or '+-'
if (*valp != 0xfffffffbU) {
return 1;
}
*valp = (uint32_t)((unsigned char)(*ss++)) - 48;
if (*valp >= 10) {
return 1;
}
}
while (!(*valp)) {
*valp = (uint32_t)((unsigned char)(*ss++)) - 48;
if ((*valp) >= 10) {
return 1;
}
}
return scan_uint_capped_finish(ss, cap, valp);
}
uint32_t scan_uint_capped(const char* ss, uint64_t cap, uint32_t* valp) {
// Reads an integer in [0, cap]. Assumes first character is nonspace.
uint32_t val = (uint32_t)((unsigned char)(*ss++)) - 48;
if (val >= 10) {
if (val != 0xfffffffbU) {
// '-' has ascii code 45, so unsigned 45 - 48 = 0xfffffffdU
if ((val != 0xfffffffdU) || (*ss != '0')) {
return 1;
}
// accept "-0", "-00", etc.
while (*(++ss) == '0');
*valp = 0;
return ((uint32_t)((unsigned char)(*ss)) - 48) < 10;
}
// accept leading '+'
val = (uint32_t)((unsigned char)(*ss++)) - 48;
if (val >= 10) {
return 1;
}
}
*valp = val;
return scan_uint_capped_finish(ss, cap, valp);
}
uint32_t scan_int_abs_bounded(const char* ss, uint64_t bound, int32_t* valp) {
// Reads an integer in [-bound, bound]. Assumes first character is nonspace.
*valp = (uint32_t)((unsigned char)(*ss++)) - 48;
int32_t sign = 1;
if (((uint32_t)*valp) >= 10) {
if (*valp == -3) {
sign = -1;
} else if (*valp != -5) {
return 1;
}
*valp = (uint32_t)((unsigned char)(*ss++)) - 48;
if (((uint32_t)*valp) >= 10) {
return 1;
}
}
if (scan_uint_capped_finish(ss, bound, (uint32_t*)valp)) {
return 1;
}
*valp *= sign;
return 0;
}
#else // not __LP64__
uint32_t scan_posint_capped32(const char* ss, uint32_t cap_div_10, uint32_t cap_mod_10, uint32_t* valp) {
// '0' has ascii code 48
uint32_t val = (uint32_t)((unsigned char)(*ss++)) - 48;
if (val >= 10) {
if (val != 0xfffffffbU) {
return 1;
}
val = (uint32_t)((unsigned char)(*ss++)) - 48;
if (val >= 10) {
return 1;
}
}
while (!val) {
val = (uint32_t)((unsigned char)(*ss++)) - 48;
if (val >= 10) {
return 1;
}
}
while (1) {
const uint32_t cur_digit = (uint32_t)((unsigned char)(*ss++)) - 48;
if (cur_digit >= 10) {
*valp = val;
return 0;
}
// avoid integer overflow in middle of computation
if ((val >= cap_div_10) && ((val > cap_div_10) || (cur_digit > cap_mod_10))) {
return 1;
}
val = val * 10 + cur_digit;
}
}
uint32_t scan_uint_capped32(const char* ss, uint32_t cap_div_10, uint32_t cap_mod_10, uint32_t* valp) {
// Reads an integer in [0, cap]. Assumes first character is nonspace.
uint32_t val = (uint32_t)((unsigned char)(*ss++)) - 48;
if (val >= 10) {
if (val != 0xfffffffbU) {
if ((val != 0xfffffffdU) || (*ss != '0')) {
return 1;
}
while (*(++ss) == '0');
*valp = 0;
return ((uint32_t)((unsigned char)(*ss)) - 48) < 10;
}
val = (uint32_t)((unsigned char)(*ss++)) - 48;
if (val >= 10) {
return 1;
}
}
while (1) {
const uint32_t cur_digit = (uint32_t)((unsigned char)(*ss++)) - 48;
if (cur_digit >= 10) {
*valp = val;
return 0;
}
if ((val >= cap_div_10) && ((val > cap_div_10) || (cur_digit > cap_mod_10))) {
return 1;
}
val = val * 10 + cur_digit;
}
}
uint32_t scan_int_abs_bounded32(const char* ss, uint32_t bound_div_10, uint32_t bound_mod_10, int32_t* valp) {
// Reads an integer in [-bound, bound]. Assumes first character is nonspace.
uint32_t val = (uint32_t)((unsigned char)(*ss++)) - 48;
int32_t sign = 1;
if (val >= 10) {
if (val == 0xfffffffdU) {
sign = -1;
} else if (val != 0xfffffffbU) {
return 1;
}
val = (uint32_t)((unsigned char)(*ss++)) - 48;
if (val >= 10) {
return 1;
}
}
while (1) {
const uint32_t cur_digit = (uint32_t)((unsigned char)(*ss++)) - 48;
if (cur_digit >= 10) {
*valp = sign * ((int32_t)val);
return 0;
}
if ((val >= bound_div_10) && ((val > bound_div_10) || (cur_digit > bound_mod_10))) {
return 1;
}
val = val * 10 + cur_digit;
}
}
#endif
uint32_t scan_posintptr(const char* ss, uintptr_t* valp) {
// Reads an integer in [1, 2^BITCT - 1]. Assumes first character is
// nonspace.
uintptr_t val = (uintptr_t)((unsigned char)(*ss++)) - 48;
if (val >= 10) {
#ifdef __LP64__
if (val != 0xfffffffffffffffbLLU) {
return 1;
}
#else
if (val != 0xfffffffbU) {
return 1;
}
#endif
val = (uintptr_t)((unsigned char)(*ss++)) - 48;
if (val >= 10) {
return 1;
}
}
while (!val) {
val = (uintptr_t)((unsigned char)(*ss++)) - 48;
if (val >= 10) {
return 1;
}
}
// limit is 20 digits, we've already read one
#ifdef __LP64__
const char* ss_limit = &(ss[20]);
#else
const char* ss_limit = &(ss[10]);
#endif
while (1) {
const uintptr_t cur_digit = (uintptr_t)((unsigned char)(*ss++)) - 48;
if (cur_digit >= 10) {
*valp = val;
return 0;
}
const uintptr_t cur_digit2 = (uintptr_t)((unsigned char)(*ss++)) - 48;
if (ss == ss_limit) {
if ((cur_digit2 < 10) || ((val >= (~ZEROLU) / 10) && ((val > (~ZEROLU) / 10) || (cur_digit > (~ZEROLU) % 10)))) {
return 1;
}
*valp = val * 10 + cur_digit;
return 0;
}
if (cur_digit2 >= 10) {
*valp = val * 10 + cur_digit;
return 0;
}
val = val * 100 + cur_digit * 10 + cur_digit2;
}
}
/*
uint32_t scan_uintptr(char* ss, uintptr_t* valp) {
// [0, 2^BITCT - 1].
uintptr_t val = (uint32_t)((unsigned char)*ss) - 48;
uintptr_t cur_digit;
if (val < 10) {
while (1) {
scan_uintptr_main_loop:
cur_digit = (uint32_t)((unsigned char)(*(++ss))) - 48;
if (cur_digit >= 10) {
*valp = val;
return 0;
}
if ((val >= (~ZEROLU) / 10) && ((val > (~ZEROLU) / 10) || (cur_digit > (~ZEROLU) % 10))) {
return 1;
}
val = val * 10 + cur_digit;
}
}
ss++;
if (val != 0xfffffffdU) {
if (val == 0xfffffffbU) {
val = (uint32_t)((unsigned char)(*ss)) - 48;
if (val < 10) {
goto scan_uintptr_main_loop;
}
}
return 1;
}
if (*ss != '0') {
return 1;
}
while (*(++ss) == '0');
*valp = 0;
return ((uint32_t)((unsigned char)(*ss)) - 48) < 10;
}
*/
uint32_t scan_posint_cappedx(const char* ss, uint64_t cap, uint32_t* valp) {
double val;
if (scan_doublex(ss, &val) || (val < 1.0) || (val > ((double)cap))) {
return 1;
}
*valp = (uint32_t)val;
return (val != ((double)(*valp)));
}
uint32_t scan_uint_cappedx(const char* ss, uint64_t cap, uint32_t* valp) {
double val;
if (scan_doublex(ss, &val) || (val < 0.0) || (val > ((double)cap))) {
return 1;
}
*valp = (uint32_t)val;
return (val != ((double)(*valp)));
}
uint32_t scan_int_abs_boundedx(const char* ss, uint64_t bound, int32_t* valp) {
const double bound_d = (double)bound;
double val;
if (scan_doublex(ss, &val) || (val < -bound_d) || (val > bound_d)) {
return 1;
}
*valp = (int32_t)val;
return (val != ((double)(*valp)));
}
uint32_t scan_posintptrx(const char* ss, uintptr_t* valp) {
double val;
if (scan_doublex(ss, &val) || (val < 1.0) || (val > ((double)(~ZEROLU)))) {
return 1;
}
*valp = (uintptr_t)val;
return (val != ((double)(*valp)));
}
uint32_t scan_two_doubles(char* ss, double* __restrict val1p, double* __restrict val2p) {
char* ss2;
*val1p = strtod(ss, &ss2);
if (ss == ss2) {
return 1;
}
ss = skip_initial_spaces(ss2);
*val2p = strtod(ss, &ss2);
return (ss == ss2)? 1 : 0;
}
int32_t scan_token_ct_len(uintptr_t half_bufsize, FILE* infile, char* buf, uintptr_t* __restrict token_ct_ptr, uintptr_t* __restrict max_token_len_ptr) {
// buf must be of size >= (2 * half_bufsize + 2)
// max_token_len includes trailing null
uintptr_t full_bufsize = half_bufsize * 2;
uintptr_t curtoklen = 0;
uintptr_t token_ct = *token_ct_ptr;
uintptr_t max_token_len = *max_token_len_ptr;
char* midbuf = &(buf[half_bufsize]);
char* bufptr;
char* bufptr2;
char* buf_end;
uintptr_t bufsize;
while (1) {
if (fread_checked(midbuf, half_bufsize, infile, &bufsize)) {
return RET_READ_FAIL;
}
if (!bufsize) {
if (curtoklen) {
// corner case
if (curtoklen >= max_token_len) {
max_token_len = curtoklen + 1;
}
token_ct++;
}
break;
}
buf_end = &(midbuf[bufsize]);
*buf_end = ' ';
buf_end[1] = '0';
bufptr = &(buf[half_bufsize - curtoklen]);
bufptr2 = midbuf;
if (curtoklen) {
goto scan_token_ct_len_tok_start;
}
while (1) {
while (*bufptr <= ' ') {
bufptr++;
}
if (bufptr >= buf_end) {
curtoklen = 0;
break;
}
bufptr2 = &(bufptr[1]);
scan_token_ct_len_tok_start:
while (*bufptr2 > ' ') {
bufptr2++;
}
curtoklen = (uintptr_t)(bufptr2 - bufptr);
if ((bufptr2 == buf_end) && (buf_end == &(buf[full_bufsize]))) {
if (curtoklen >= half_bufsize) {
return RET_INVALID_FORMAT;
}
break;
}
if (curtoklen >= max_token_len) {
if (curtoklen >= half_bufsize) {
return RET_INVALID_FORMAT;
}
max_token_len = curtoklen + 1;
}
token_ct++;
bufptr = &(bufptr2[1]);
}
}
if (!feof(infile)) {
return RET_READ_FAIL;
}
*max_token_len_ptr = max_token_len;
*token_ct_ptr = token_ct;
return 0;
}
int32_t read_tokens(uintptr_t half_bufsize, uintptr_t token_ct, uintptr_t max_token_len, FILE* infile, char* __restrict buf, char* __restrict token_name_buf) {
// buf must be of size >= (2 * half_bufsize + 2).
// max_token_len includes trailing null
uintptr_t full_bufsize = half_bufsize * 2;
uintptr_t curtoklen = 0;
uintptr_t token_idx = 0;
char* midbuf = &(buf[half_bufsize]);
char* bufptr = midbuf;
char* bufptr2;
char* bufptr3;
char* buf_end;
uintptr_t bufsize;
while (1) {
if (fread_checked(midbuf, half_bufsize, infile, &bufsize)) {
return RET_READ_FAIL;
}
if (!bufsize) {
if (curtoklen) {
if (token_idx + 1 == token_ct) {
memcpyx(&(token_name_buf[token_idx * max_token_len]), bufptr, curtoklen, '\0');
return 0;
}
}
// something very strange has to happen to get here
return RET_READ_FAIL;
}
buf_end = &(midbuf[bufsize]);
*buf_end = ' ';
buf_end[1] = '0';
bufptr2 = midbuf;
if (curtoklen) {
goto read_tokens_tok_start;
}
while (1) {
while (*bufptr <= ' ') {
bufptr++;
}
if (bufptr >= buf_end) {
curtoklen = 0;
bufptr = midbuf;
break;
}
bufptr2 = &(bufptr[1]);
read_tokens_tok_start:
while (*bufptr2 > ' ') {
bufptr2++;
}
curtoklen = (uintptr_t)(bufptr2 - bufptr);
if ((bufptr2 == buf_end) && (buf_end == &(buf[full_bufsize]))) {
bufptr3 = &(buf[half_bufsize - curtoklen]);
memcpy(bufptr3, bufptr, curtoklen);
bufptr = bufptr3;
break;
}
memcpyx(&(token_name_buf[token_idx * max_token_len]), bufptr, curtoklen, '\0');
if (++token_idx == token_ct) {
return 0;
}
bufptr = &(bufptr2[1]);
}
}
}
int32_t gzputs_w4(gzFile gz_outfile, const char* ss) {
if (!ss[1]) {
if (gzputs(gz_outfile, " ") == -1) {
return -1;
}
return gzputc(gz_outfile, ss[0]);
}
if (!ss[2]) {
if (gzputs(gz_outfile, " ") == -1) {
return -1;
}
} else if (!ss[3]) {
if (gzputc(gz_outfile, ' ') == -1) {
return -1;
}
}
return gzputs(gz_outfile, ss);
}
int32_t get_next_noncomment(FILE* fptr, char** lptr_ptr, uintptr_t* line_idx_ptr) {
char* lptr;
do {
if (!fgets(g_textbuf, MAXLINELEN, fptr)) {
return -1;
}
*line_idx_ptr += 1;
lptr = skip_initial_spaces(g_textbuf);
} while (is_eoln_or_comment_kns(*lptr));
*lptr_ptr = lptr;
return 0;
}
int32_t get_next_noncomment_excl(const uintptr_t* __restrict marker_exclude, FILE* fptr, char** lptr_ptr, uintptr_t* __restrict line_idx_ptr, uintptr_t* __restrict marker_uidx_ptr) {
while (!get_next_noncomment(fptr, lptr_ptr, line_idx_ptr)) {
if (!is_set_ul(marker_exclude, *marker_uidx_ptr)) {
return 0;
}
*marker_uidx_ptr += 1;
}
return -1;
}
void get_top_two_ui(const uint32_t* __restrict uint_arr, uintptr_t uia_size, uintptr_t* __restrict top_idx_ptr, uintptr_t* __restrict second_idx_ptr) {
assert(uia_size > 1);
uintptr_t top_idx = (uint_arr[1] > uint_arr[0])? 1 : 0;
uintptr_t second_idx = 1 ^ top_idx;
uint32_t top_val = uint_arr[top_idx];
uint32_t second_val = uint_arr[second_idx];
uintptr_t cur_idx;
uintptr_t cur_val;
for (cur_idx = 2; cur_idx < uia_size; ++cur_idx) {
cur_val = uint_arr[cur_idx];
if (cur_val > second_val) {
if (cur_val > top_val) {
second_val = top_val;
second_idx = top_idx;
top_val = cur_val;
top_idx = cur_idx;
} else {
second_val = cur_val;
second_idx = cur_idx;
}
}
}
*top_idx_ptr = top_idx;
*second_idx_ptr = second_idx;
}
uint32_t intlen(int32_t num) {
int32_t retval = 1;
uint32_t absnum;
if (num < 0) {
absnum = -num;
retval++;
} else {
absnum = num;
}
while (absnum > 99) {
// division by a constant is faster for unsigned ints
absnum /= 100;
retval += 2;
}
if (absnum > 9) {
retval++;
}
return retval;
}
int32_t strcmp_se(const char* s_read, const char* s_const, uint32_t s_const_len) {
return memcmp(s_read, s_const, s_const_len) || (!is_space_or_eoln(s_read[s_const_len]));
}
char* next_token(char* sptr) {
if (!sptr) {
return nullptr;
}
unsigned char ucc = *sptr;
while (ucc > 32) {
ucc = *(++sptr);
}
while ((ucc == ' ') || (ucc == '\t')) {
ucc = *(++sptr);
}
return (ucc > 32)? sptr : nullptr;
}
char* next_token_mult(char* sptr, uint32_t ct) {
assert(ct);
if (!sptr) {
return nullptr;
}
unsigned char ucc = *sptr;
do {
while (ucc > 32) {
ucc = *(++sptr);
}
while ((ucc == ' ') || (ucc == '\t')) {
ucc = *(++sptr);
}
if (ucc <= 32) {
return nullptr;
}
} while (--ct);
return sptr;
}
uint32_t count_tokens(const char* bufptr) {
uint32_t token_ct = 0;
while ((*bufptr == ' ') || (*bufptr == '\t')) {
bufptr++;
}
while (!is_eoln_kns(*bufptr)) {
token_ct++;
while (!is_space_or_eoln(*(++bufptr)));
while ((*bufptr == ' ') || (*bufptr == '\t')) {
bufptr++;
}
}
return token_ct;
}
uint32_t count_and_measure_multistr(const char* multistr, uintptr_t* max_slen_ptr) {
// max_slen includes null terminator
// assumes multistr is nonempty
uint32_t ct = 0;
uintptr_t max_slen = *max_slen_ptr;
uintptr_t slen;
do {
slen = strlen(multistr) + 1;
if (slen > max_slen) {
max_slen = slen;
}
multistr = &(multistr[slen]);
ct++;
} while (*multistr);
*max_slen_ptr = max_slen;
return ct;
}
// number-to-string encoders
static const char digit2_table[200] = {
'0', '0', '0', '1', '0', '2', '0', '3', '0', '4',
'0', '5', '0', '6', '0', '7', '0', '8', '0', '9',
'1', '0', '1', '1', '1', '2', '1', '3', '1', '4',
'1', '5', '1', '6', '1', '7', '1', '8', '1', '9',
'2', '0', '2', '1', '2', '2', '2', '3', '2', '4',
'2', '5', '2', '6', '2', '7', '2', '8', '2', '9',
'3', '0', '3', '1', '3', '2', '3', '3', '3', '4',
'3', '5', '3', '6', '3', '7', '3', '8', '3', '9',
'4', '0', '4', '1', '4', '2', '4', '3', '4', '4',
'4', '5', '4', '6', '4', '7', '4', '8', '4', '9',
'5', '0', '5', '1', '5', '2', '5', '3', '5', '4',
'5', '5', '5', '6', '5', '7', '5', '8', '5', '9',
'6', '0', '6', '1', '6', '2', '6', '3', '6', '4',
'6', '5', '6', '6', '6', '7', '6', '8', '6', '9',
'7', '0', '7', '1', '7', '2', '7', '3', '7', '4',
'7', '5', '7', '6', '7', '7', '7', '8', '7', '9',
'8', '0', '8', '1', '8', '2', '8', '3', '8', '4',
'8', '5', '8', '6', '8', '7', '8', '8', '8', '9',
'9', '0', '9', '1', '9', '2', '9', '3', '9', '4',
'9', '5', '9', '6', '9', '7', '9', '8', '9', '9'};
char* uint32toa(uint32_t uii, char* start) {
// Memory-efficient fast integer writer. (You can do a bit better sometimes
// by using a larger lookup table, but on average I doubt that pays off.)
// Returns a pointer to the end of the integer (not null-terminated).
uint32_t quotient;
if (uii < 1000) {
if (uii < 10) {
*start++ = '0' + uii;
return start;
}
if (uii < 100) {
goto uint32toa_2;
}
quotient = uii / 100;
*start++ = '0' + quotient;
} else {
if (uii < 10000000) {
if (uii >= 100000) {
if (uii < 1000000) {
goto uint32toa_6;
}
quotient = uii / 1000000;
*start++ = '0' + quotient;
goto uint32toa_6b;
}
if (uii < 10000) {
goto uint32toa_4;
}
quotient = uii / 10000;
*start++ = '0' + quotient;
} else {
if (uii >= 100000000) {
quotient = uii / 100000000;
if (uii >= 1000000000) {
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
} else {
*start++ = '0' + quotient;
}
uii -= 100000000 * quotient;
}
quotient = uii / 1000000;
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
uint32toa_6b:
uii -= 1000000 * quotient;
uint32toa_6:
quotient = uii / 10000;
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
}
uii -= 10000 * quotient;
uint32toa_4:
// could make a uitoa_z4() call here, but that's slightly slower
quotient = uii / 100;
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
}
uii -= 100 * quotient;
uint32toa_2:
return memcpya(start, &(digit2_table[uii * 2]), 2);
}
char* int32toa(int32_t ii, char* start) {
uint32_t uii = ii;
if (ii < 0) {
// -INT_MIN is undefined, but negating the unsigned int equivalent works
*start++ = '-';
uii = -uii;
}
return uint32toa(uii, start);
}
char* uitoa_z4(uint32_t uii, char* start) {
uint32_t quotient = uii / 100;
assert(quotient < 100);
uii -= 100 * quotient;
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
return memcpya(start, &(digit2_table[uii * 2]), 2);
}
char* uitoa_z6(uint32_t uii, char* start) {
uint32_t quotient = uii / 10000;
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
return uitoa_z4(uii - 10000 * quotient, start);
}
char* uitoa_z8(uint32_t uii, char* start) {
uint32_t quotient = uii / 1000000;
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
return uitoa_z6(uii - 1000000 * quotient, start);
}
char* int64toa(int64_t llii, char* start) {
uint64_t ullii = llii;
uint64_t top_digits;
uint32_t bottom_eight;
uint32_t middle_eight;
if (llii < 0) {
*start++ = '-';
ullii = -ullii;
}
if (ullii <= 0xffffffffLLU) {
return uint32toa((uint32_t)ullii, start);
}
top_digits = ullii / 100000000;
bottom_eight = (uint32_t)(ullii - (top_digits * 100000000));
if (top_digits <= 0xffffffffLLU) {
start = uint32toa((uint32_t)top_digits, start);
return uitoa_z8(bottom_eight, start);
}
ullii = top_digits / 100000000;
middle_eight = (uint32_t)(top_digits - (ullii * 100000000));
start = uint32toa((uint32_t)ullii, start);
start = uitoa_z8(middle_eight, start);
return uitoa_z8(bottom_eight, start);
}
char* uint32toa_w4(uint32_t uii, char* start) {
uint32_t quotient;
if (uii < 1000) {
if (uii < 10) {
// assumes little-endian
*((uint32_t*)start) = 0x30202020 + (uii << 24);
return &(start[4]);
}
if (uii < 100) {
memset(start, 32, 2);
} else {
quotient = uii / 100;
*start++ = ' ';
*start++ = '0' + quotient;
uii -= quotient * 100;
}
return memcpya(start, &(digit2_table[uii * 2]), 2);
} else {
// presumably the field width is 4 for a reason; don't bother optimizing
// this
return uint32toa(uii, start);
}
}
char* uint32toa_w6(uint32_t uii, char* start) {
uint32_t quotient;
if (uii < 1000) {
if (uii < 10) {
start = memseta(start, 32, 5);
*start++ = '0' + uii;
return start;
}
if (uii < 100) {
start = memseta(start, 32, 4);
goto uint32toa_w6_2;
}
quotient = uii / 100;
// the little-endian trick doesn't seem to help here. possibly relevant
// differences from uint32toa_w4() and _w8(): sequential dependence on
// quotient, need to interpret pointer as a char* again
start = memseta(start, 32, 3);
*start++ = '0' + quotient;
} else {
if (uii < 10000000) {
if (uii >= 100000) {
if (uii < 1000000) {
goto uint32toa_w6_6;
}
quotient = uii / 1000000;
*start++ = '0' + quotient;
goto uint32toa_w6_6b;
} else if (uii >= 10000) {
*start++ = ' ';
quotient = uii / 10000;
*start++ = '0' + quotient;
} else {
start = memseta(start, 32, 2);
goto uint32toa_w6_4;
}
} else {
if (uii >= 100000000) {
quotient = uii / 100000000;
if (uii >= 1000000000) {
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
} else {
*start++ = '0' + quotient;
}
uii -= 100000000 * quotient;
}
quotient = uii / 1000000;
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
uint32toa_w6_6b:
uii -= 1000000 * quotient;
uint32toa_w6_6:
quotient = uii / 10000;
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
}
uii -= 10000 * quotient;
uint32toa_w6_4:
quotient = uii / 100;
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
}
uii -= 100 * quotient;
uint32toa_w6_2:
return memcpya(start, &(digit2_table[uii * 2]), 2);
}
char* uint32toa_w7(uint32_t uii, char* start) {
uint32_t quotient;
if (uii < 1000) {
if (uii < 10) {
start = memseta(start, 32, 6);
*start++ = '0' + uii;
return start;
}
if (uii < 100) {
start = memseta(start, 32, 5);
goto uint32toa_w7_2;
}
quotient = uii / 100;
start = memseta(start, 32, 4);
*start++ = '0' + quotient;
} else {
if (uii < 10000000) {
if (uii >= 100000) {
if (uii >= 1000000) {
quotient = uii / 1000000;
*start++ = '0' + quotient;
goto uint32toa_w7_6b;
}
*start++ = ' ';
goto uint32toa_w7_6;
} else if (uii >= 10000) {
start = memseta(start, 32, 2);
quotient = uii / 10000;
*start++ = '0' + quotient;
} else {
start = memseta(start, 32, 3);
goto uint32toa_w7_4;
}
} else {
if (uii >= 100000000) {
quotient = uii / 100000000;
if (uii >= 1000000000) {
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
} else {
*start++ = '0' + quotient;
}
uii -= 100000000 * quotient;
}
quotient = uii / 1000000;
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
uint32toa_w7_6b:
uii -= 1000000 * quotient;
uint32toa_w7_6:
quotient = uii / 10000;
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
}
uii -= 10000 * quotient;
uint32toa_w7_4:
quotient = uii / 100;
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
}
uii -= 100 * quotient;
uint32toa_w7_2:
return memcpya(start, &(digit2_table[uii * 2]), 2);
}
char* uint32toa_w8(uint32_t uii, char* start) {
uint32_t quotient;
if (uii < 1000) {
if (uii < 10) {
#ifdef __LP64__
*((uintptr_t*)start) = 0x3020202020202020LLU + (((uintptr_t)uii) << 56);
return &(start[8]);
#else
start = memseta(start, 32, 7);
*start++ = '0' + uii;
return start;
#endif
}
if (uii < 100) {
start = memseta(start, 32, 6);
goto uint32toa_w8_2;
}
quotient = uii / 100;
start = memseta(start, 32, 5);
*start++ = '0' + quotient;
} else {
if (uii < 10000000) {
if (uii >= 100000) {
if (uii < 1000000) {
start = memseta(start, 32, 2);
goto uint32toa_w8_6;
}
quotient = uii / 1000000;
*start = ' ';
start[1] = '0' + quotient;
start += 2;
goto uint32toa_w8_6b;
} else if (uii < 10000) {
start = memseta(start, 32, 4);
goto uint32toa_w8_4;
}
memset(start, 32, 3);
quotient = uii / 10000;
start[3] = '0' + quotient;
start += 4;
} else {
if (uii >= 100000000) {
quotient = uii / 100000000;
if (uii >= 1000000000) {
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
} else {
*start++ = '0' + quotient;
}
uii -= 100000000 * quotient;
}
quotient = uii / 1000000;
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
uint32toa_w8_6b:
uii -= 1000000 * quotient;
uint32toa_w8_6:
quotient = uii / 10000;
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
}
uii -= 10000 * quotient;
uint32toa_w8_4:
quotient = uii / 100;
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
}
uii -= 100 * quotient;
uint32toa_w8_2:
return memcpya(start, &(digit2_table[uii * 2]), 2);
}
char* uint32toa_w10(uint32_t uii, char* start) {
// if we decide to reduce code size and optimize only one field width, this
// should be it
uint32_t quotient;
if (uii < 1000) {
if (uii < 10) {
start = memseta(start, 32, 9);
*start++ = '0' + uii;
return start;
}
if (uii < 100) {
start = memseta(start, 32, 8);
goto uint32toa_w10_2;
}
quotient = uii / 100;
start = memseta(start, 32, 7);
*start++ = '0' + quotient;
} else {
if (uii < 10000000) {
if (uii >= 100000) {
if (uii < 1000000) {
start = memseta(start, 32, 4);
goto uint32toa_w10_6;
}
quotient = uii / 1000000;
memset(start, 32, 3);
start[3] = '0' + quotient;
start += 4;
goto uint32toa_w10_6b;
} else if (uii < 10000) {
start = memseta(start, 32, 6);
goto uint32toa_w10_4;
}
memset(start, 32, 5);
quotient = uii / 10000;
start[5] = '0' + quotient;
start += 6;
} else {
if (uii >= 100000000) {
quotient = uii / 100000000;
if (uii >= 1000000000) {
memcpy(start, &(digit2_table[quotient * 2]), 2);
} else {
*start = ' ';
start[1] = '0' + quotient;
}
uii -= 100000000 * quotient;
} else {
memset(start, 32, 2);
}
quotient = uii / 1000000;
memcpy(&(start[2]), &(digit2_table[quotient * 2]), 2);
start += 4;
uint32toa_w10_6b:
uii -= 1000000 * quotient;
uint32toa_w10_6:
quotient = uii / 10000;
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
}
uii -= 10000 * quotient;
uint32toa_w10_4:
quotient = uii / 100;
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
}
uii -= 100 * quotient;
uint32toa_w10_2:
return memcpya(start, &(digit2_table[uii * 2]), 2);
}
static inline char* uitoa_trunc2(uint32_t uii, char* start) {
// Given 0 < uii < 100, writes uii without *trailing* zeroes. (I.e. this is
// for floating-point encoder use.)
memcpy(start, &(digit2_table[uii * 2]), 2);
if (start[1] != '0') {
return &(start[2]);
}
return &(start[1]);
}
static inline char* uitoa_trunc3(uint32_t uii, char* start) {
*start++ = '0' + (uii / 100);
uii %= 100;
if (!uii) {
return start;
}
memcpy(start, &(digit2_table[uii * 2]), 2);
if (start[1] != '0') {
return &(start[2]);
}
return &(start[1]);
}
static inline char* uitoa_trunc4(uint32_t uii, char* start) {
uint32_t quotient = uii / 100;
memcpy(start, &(digit2_table[quotient * 2]), 2);
uii -= 100 * quotient;
if (uii) {
start += 2;
memcpy(start, &(digit2_table[uii * 2]), 2);
}
if (start[1] != '0') {
return &(start[2]);
}
return &(start[1]);
}
static inline char* uitoa_trunc6(uint32_t uii, char* start) {
uint32_t quotient = uii / 10000;
memcpy(start, &(digit2_table[quotient * 2]), 2);
uii -= 10000 * quotient;
if (uii) {
quotient = uii / 100;
start += 2;
memcpy(start, &(digit2_table[quotient * 2]), 2);
uii -= 100 * quotient;
if (uii) {
start += 2;
memcpy(start, &(digit2_table[uii * 2]), 2);
}
}
if (start[1] != '0') {
return &(start[2]);
}
return &(start[1]);
}
static inline char* uitoa_trunc8(uint32_t uii, char* start) {
uint32_t quotient = uii / 1000000;
memcpy(start, &(digit2_table[quotient * 2]), 2);
uii -= 1000000 * quotient;
if (uii) {
quotient = uii / 10000;
start += 2;
memcpy(start, &(digit2_table[quotient * 2]), 2);
uii -= 10000 * quotient;
if (uii) {
quotient = uii / 100;
start += 2;
memcpy(start, &(digit2_table[quotient * 2]), 2);
uii -= 100 * quotient;
if (uii) {
start += 2;
memcpy(start, &(digit2_table[uii * 2]), 2);
}
}
}
if (start[1] != '0') {
return &(start[2]);
}
return &(start[1]);
}
static inline char* qrtoa_1p1(uint32_t quotient, uint32_t remainder, char* start) {
start[0] = '0' + quotient;
if (!remainder) {
return &(start[1]);
}
start[1] = '.';
start[2] = '0' + remainder;
return &(start[3]);
}
static inline char* qrtoa_1p2(uint32_t quotient, uint32_t remainder, char* start) {
*start++ = '0' + quotient;
if (!remainder) {
return start;
}
*start++ = '.';
memcpy(start, &(digit2_table[remainder * 2]), 2);
if (start[1] != '0') {
return &(start[2]);
}
return &(start[1]);
}
static inline char* qrtoa_1p3(uint32_t quotient, uint32_t remainder, char* start) {
// quotient = (int32_t)dxx;
// remainder = ((int32_t)(dxx * 1000)) - (quotient * 1000);
*start++ = '0' + quotient;
if (!remainder) {
return start;
}
*start++ = '.';
quotient = remainder / 10;
memcpy(start, &(digit2_table[quotient * 2]), 2);
remainder -= 10 * quotient;
if (remainder) {
start[2] = '0' + remainder;
return &(start[3]);
}
if (start[1] != '0') {
return &(start[2]);
}
return &(start[1]);
}
static inline char* qrtoa_1p5(uint32_t quotient, uint32_t remainder, char* start) {
*start++ = '0' + quotient;
if (!remainder) {
return start;
}
*start++ = '.';
quotient = remainder / 1000;
memcpy(start, &(digit2_table[quotient * 2]), 2);
remainder -= 1000 * quotient;
if (remainder) {
quotient = remainder / 10;
start += 2;
memcpy(start, &(digit2_table[quotient * 2]), 2);
remainder -= 10 * quotient;
if (remainder) {
start[2] = '0' + remainder;
return &(start[3]);
}
}
if (start[1] != '0') {
return &(start[2]);
}
return &(start[1]);
}
static inline char* qrtoa_1p7(uint32_t quotient, uint32_t remainder, char* start) {
*start++ = '0' + quotient;
if (!remainder) {
return start;
}
*start++ = '.';
quotient = remainder / 100000;
memcpy(start, &(digit2_table[quotient * 2]), 2);
remainder -= 100000 * quotient;
if (remainder) {
quotient = remainder / 1000;
start += 2;
memcpy(start, &(digit2_table[quotient * 2]), 2);
remainder -= 1000 * quotient;
if (remainder) {
quotient = remainder / 10;
start += 2;
memcpy(start, &(digit2_table[quotient * 2]), 2);
remainder -= 10 * quotient;
if (remainder) {
start[2] = '0' + remainder;
return &(start[3]);
}
}
}
if (start[1] != '0') {
return &(start[2]);
}
return &(start[1]);
}
// Okay, time to do banker's rounding when printing doubles. 14 digits of
// precision are used in judging equality to 0.5 (actual precision of doubles
// is 15-17 digits); the intention is to capture all directly loaded or exactly
// computed edge cases (so enough tolerance is needed to survive the internal
// multiplications by powers of 10, etc.), while rounding a negligible number
// of honest-to-god 0.4999999s up and 0.5000001s down.
// To avoid inadvertent printing of an extra digit, there's a deliberate gap
// between the 99.9994999...-type bounds and the largest numbers that would
// actually round down.
static const double banker_round5[] = {0.499995, 0.500005};
static const double banker_round6[] = {0.4999995, 0.5000005};
static const double banker_round7[] = {0.49999995, 0.50000005};
static const double banker_round8[] = {0.499999995, 0.500000005};
static const double banker_round9[] = {0.4999999995, 0.5000000005};
static const double banker_round10[] = {0.49999999995, 0.50000000005};
static const double banker_round11[] = {0.499999999995, 0.500000000005};
static const double banker_round12[] = {0.4999999999995, 0.5000000000005};
static inline uint32_t double_bround(double dxx, const double* banker_round) {
uint32_t result = (int32_t)dxx;
return result + (int32_t)((dxx - ((int32_t)result)) + banker_round[result & 1]);
}
// These are separate functions so the compiler can optimize the integer
// divisions.
static inline void double_bround1(double dxx, const double* banker_round, uint32_t* quotientp, uint32_t* remainderp) {
dxx *= 10;
uint32_t remainder = (int32_t)dxx;
remainder += (int32_t)((dxx - ((int32_t)remainder)) + banker_round[remainder & 1]);
*quotientp = remainder / 10;
*remainderp = remainder - (*quotientp) * 10;
}
static inline void double_bround2(double dxx, const double* banker_round, uint32_t* quotientp, uint32_t* remainderp) {
dxx *= 100;
uint32_t remainder = (int32_t)dxx;
remainder += (int32_t)((dxx - ((int32_t)remainder)) + banker_round[remainder & 1]);
*quotientp = remainder / 100;
*remainderp = remainder - (*quotientp) * 100;
}
static inline void double_bround3(double dxx, const double* banker_round, uint32_t* quotientp, uint32_t* remainderp) {
dxx *= 1000;
uint32_t remainder = (int32_t)dxx;
remainder += (int32_t)((dxx - ((int32_t)remainder)) + banker_round[remainder & 1]);
*quotientp = remainder / 1000;
*remainderp = remainder - (*quotientp) * 1000;
}
static inline void double_bround4(double dxx, const double* banker_round, uint32_t* quotientp, uint32_t* remainderp) {
dxx *= 10000;
uint32_t remainder = (int32_t)dxx;
remainder += (int32_t)((dxx - ((int32_t)remainder)) + banker_round[remainder & 1]);
*quotientp = remainder / 10000;
*remainderp = remainder - (*quotientp) * 10000;
}
static inline void double_bround5(double dxx, const double* banker_round, uint32_t* quotientp, uint32_t* remainderp) {
dxx *= 100000;
uint32_t remainder = (int32_t)dxx;
remainder += (int32_t)((dxx - ((int32_t)remainder)) + banker_round[remainder & 1]);
*quotientp = remainder / 100000;
*remainderp = remainder - (*quotientp) * 100000;
}
static inline void double_bround6(double dxx, const double* banker_round, uint32_t* quotientp, uint32_t* remainderp) {
dxx *= 1000000;
uint32_t remainder = (int32_t)dxx;
remainder += (int32_t)((dxx - ((int32_t)remainder)) + banker_round[remainder & 1]);
*quotientp = remainder / 1000000;
*remainderp = remainder - (*quotientp) * 1000000;
}
static inline void double_bround7(double dxx, const double* banker_round, uint32_t* quotientp, uint32_t* remainderp) {
dxx *= 10000000;
uint32_t remainder = (int32_t)dxx;
remainder += (int32_t)((dxx - ((int32_t)remainder)) + banker_round[remainder & 1]);
*quotientp = remainder / 10000000;
*remainderp = remainder - (*quotientp) * 10000000;
}
char* dtoa_so6(double dxx, char* start) {
// 6 sig fig number, 0.999995 <= dxx < 999999.5
// 'so' = "significand only"
// Just hardcoding all six cases, in the absence of a better approach...
uint32_t uii;
uint32_t quotient;
uint32_t remainder;
if (dxx < 99.999949999999) {
if (dxx < 9.9999949999999) {
double_bround5(dxx, banker_round8, "ient, &remainder);
return qrtoa_1p5(quotient, remainder, start);
}
double_bround4(dxx, banker_round8, "ient, &remainder);
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
if (!remainder) {
return start;
}
*start++ = '.';
quotient = remainder / 100;
memcpy(start, &(digit2_table[quotient * 2]), 2);
remainder -= 100 * quotient;
if (remainder) {
start += 2;
dtoa_so6_pretail:
memcpy(start, &(digit2_table[remainder * 2]), 2);
}
dtoa_so6_tail:
if (start[1] != '0') {
return &(start[2]);
}
return &(start[1]);
} else if (dxx < 9999.9949999999) {
if (dxx < 999.99949999999) {
double_bround3(dxx, banker_round8, &uii, &remainder);
quotient = uii / 100;
*start++ = '0' + quotient;
quotient = uii - 100 * quotient;
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
if (!remainder) {
return start;
}
*start++ = '.';
quotient = remainder / 10;
memcpy(start, &(digit2_table[quotient * 2]), 2);
remainder -= quotient * 10;
if (!remainder) {
goto dtoa_so6_tail;
}
start[2] = '0' + remainder;
return &(start[3]);
}
double_bround2(dxx, banker_round8, &uii, &remainder);
quotient = uii / 100;
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
quotient = uii - (100 * quotient);
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
if (!remainder) {
return start;
}
*start++ = '.';
goto dtoa_so6_pretail;
} else if (dxx < 99999.949999999) {
double_bround1(dxx, banker_round8, &uii, &remainder);
quotient = uii / 10000;
*start = '0' + quotient;
uii -= 10000 * quotient;
quotient = uii / 100;
start = memcpya(&(start[1]), &(digit2_table[quotient * 2]), 2);
uii = uii - 100 * quotient;
start = memcpya(start, &(digit2_table[uii * 2]), 2);
if (!remainder) {
return start;
}
*start++ = '.';
*start = '0' + remainder;
return &(start[1]);
} else {
return uitoa_z6(double_bround(dxx, banker_round8), start);
}
}
// Briefly had banker's rounding for floats, but then I realized that the only
// float-printing function calls are --make-grm related, they all request 6-7
// digits of precision, and at that point it's impossible to distinguish exact
// 0.5-matches in the remainder. So we just have generic rounding functions
// here, with similar interfaces to the double-rounding functions to minimize
// the need for separate reasoning about this code.
static inline uint32_t float_round(float fxx) {
return (uint32_t)((int32_t)(fxx + 0.5));
}
static inline void float_round1(float fxx, uint32_t* quotientp, uint32_t* remainderp) {
uint32_t remainder = float_round(fxx * 10);
*quotientp = remainder / 10;
*remainderp = remainder - (*quotientp) * 10;
}
static inline void float_round2(float fxx, uint32_t* quotientp, uint32_t* remainderp) {
uint32_t remainder = float_round(fxx * 100);
*quotientp = remainder / 100;
*remainderp = remainder - (*quotientp) * 100;
}
static inline void float_round3(float fxx, uint32_t* quotientp, uint32_t* remainderp) {
uint32_t remainder = float_round(fxx * 1000);
*quotientp = remainder / 1000;
*remainderp = remainder - (*quotientp) * 1000;
}
static inline void float_round4(float fxx, uint32_t* quotientp, uint32_t* remainderp) {
uint32_t remainder = float_round(fxx * 10000);
*quotientp = remainder / 10000;
*remainderp = remainder - (*quotientp) * 10000;
}
static inline void float_round5(float fxx, uint32_t* quotientp, uint32_t* remainderp) {
uint32_t remainder = float_round(fxx * 100000);
*quotientp = remainder / 100000;
*remainderp = remainder - (*quotientp) * 100000;
}
static inline void float_round6(float fxx, uint32_t* quotientp, uint32_t* remainderp) {
uint32_t remainder = float_round(fxx * 1000000);
*quotientp = remainder / 1000000;
*remainderp = remainder - (*quotientp) * 1000000;
}
char* ftoa_so6(float fxx, char* start) {
uint32_t uii;
uint32_t quotient;
uint32_t remainder;
// difference between consecutive floats near 10 can be as large as
// 10 * 2^{-23}, which is just under 1.2e-6. So, to avoid printing an extra
// digit, we have to set this bound to be robust to an addition error of size
// 6e-7.
// (possible todo: just brute-force test this on all <2^32 possible floats
// and look for a better threshold)
if (fxx < 99.999944) {
if (fxx < 9.9999944) {
float_round5(fxx, "ient, &remainder);
return qrtoa_1p5(quotient, remainder, start);
}
float_round4(fxx, "ient, &remainder);
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
if (!remainder) {
return start;
}
*start++ = '.';
quotient = remainder / 100;
memcpy(start, &(digit2_table[quotient * 2]), 2);
remainder -= 100 * quotient;
if (remainder) {
start += 2;
ftoa_so6_pretail:
memcpy(start, &(digit2_table[remainder * 2]), 2);
}
ftoa_so6_tail:
if (start[1] != '0') {
return &(start[2]);
}
return &(start[1]);
} else if (fxx < 9999.9944) {
if (fxx < 999.99944) {
float_round3(fxx, &uii, &remainder);
quotient = uii / 100;
*start = '0' + quotient;
quotient = uii - 100 * quotient;
start = memcpya(&(start[1]), &(digit2_table[quotient * 2]), 2);
if (!remainder) {
return start;
}
*start++ = '.';
quotient = remainder / 10;
memcpy(start, &(digit2_table[quotient * 2]), 2);
remainder -= quotient * 10;
if (!remainder) {
goto ftoa_so6_tail;
}
start[2] = '0' + remainder;
return &(start[3]);
}
float_round2(fxx, &uii, &remainder);
quotient = uii / 100;
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
quotient = uii - (100 * quotient);
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
if (!remainder) {
return start;
}
*start++ = '.';
goto ftoa_so6_pretail;
} else if (fxx < 99999.944) {
float_round1(fxx, &uii, &remainder);
quotient = uii / 10000;
*start = '0' + quotient;
uii -= 10000 * quotient;
quotient = uii / 100;
start = memcpya(&(start[1]), &(digit2_table[quotient * 2]), 2);
uii = uii - 100 * quotient;
start = memcpya(start, &(digit2_table[uii * 2]), 2);
if (!remainder) {
return start;
}
*start = '.';
start[1] = '0' + remainder;
return &(start[2]);
} else {
return uitoa_z6(float_round(fxx), start);
}
}
char* dtoa_so2(double dxx, char* start) {
// 2 sig fig number, 0.95 <= dxx < 99.5
uint32_t quotient;
uint32_t remainder;
if (dxx < 9.9499999999999) {
double_bround1(dxx, banker_round12, "ient, &remainder);
return qrtoa_1p1(quotient, remainder, start);
}
return memcpya(start, &(digit2_table[(double_bround(dxx, banker_round12)) * 2]), 2);
}
char* dtoa_so3(double dxx, char* start) {
// 3 sig fig number, 0.995 <= dxx < 999.5
uint32_t quotient;
uint32_t remainder;
if (dxx < 99.949999999999) {
if (dxx < 9.9949999999999) {
double_bround2(dxx, banker_round11, "ient, &remainder);
return qrtoa_1p2(quotient, remainder, start);
}
double_bround1(dxx, banker_round11, "ient, &remainder);
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
if (!remainder) {
return start;
}
*start++ = '.';
} else {
quotient = double_bround(dxx, banker_round11);
start = memcpya(start, &(digit2_table[(quotient / 10) * 2]), 2);
remainder = quotient % 10;
}
*start = '0' + remainder;
return &(start[1]);
}
char* dtoa_so4(double dxx, char* start) {
// 4 sig fig number, 0.9995 <= dxx < 9999.5
uint32_t uii;
uint32_t quotient;
uint32_t remainder;
if (dxx < 99.994999999999) {
if (dxx < 9.9994999999999) {
double_bround3(dxx, banker_round10, "ient, &remainder);
return qrtoa_1p3(quotient, remainder, start);
}
double_bround2(dxx, banker_round10, "ient, &remainder);
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
if (!remainder) {
return start;
}
*start++ = '.';
memcpy(start, &(digit2_table[remainder * 2]), 2);
if (start[1] != '0') {
return &(start[2]);
}
return &(start[1]);
} else if (dxx < 999.94999999999) {
double_bround1(dxx, banker_round10, &uii, &remainder);
quotient = uii / 100;
*start = '0' + quotient;
quotient = uii - 100 * quotient;
start = memcpya(&(start[1]), &(digit2_table[quotient * 2]), 2);
if (!remainder) {
return start;
}
*start = '.';
start[1] = '0' + remainder;
return &(start[2]);
} else {
uitoa_z4(double_bround(dxx, banker_round10), start);
return &(start[4]);
}
}
char* dtoa_so8(double dxx, char* start) {
// 8 sig fig number, 0.99999995 <= dxx < 99999999.5
uint32_t uii;
uint32_t quotient;
uint32_t remainder;
if (dxx < 99.999999499999) {
if (dxx < 9.9999999499999) {
double_bround7(dxx, banker_round6, "ient, &remainder);
return qrtoa_1p7(quotient, remainder, start);
}
double_bround6(dxx, banker_round6, "ient, &remainder);
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
if (!remainder) {
return start;
}
*start++ = '.';
quotient = remainder / 10000;
memcpy(start, &(digit2_table[quotient * 2]), 2);
remainder -= 10000 * quotient;
if (remainder) {
start += 2;
dtoa_so8_pretail4:
quotient = remainder / 100;
memcpy(start, &(digit2_table[quotient * 2]), 2);
remainder -= 100 * quotient;
if (remainder) {
start += 2;
dtoa_so8_pretail2:
memcpy(start, &(digit2_table[remainder * 2]), 2);
}
}
dtoa_so8_tail:
if (start[1] != '0') {
return &(start[2]);
}
return &(start[1]);
} else if (dxx < 9999.9999499999) {
if (dxx < 999.99999499999) {
double_bround5(dxx, banker_round6, &uii, &remainder);
quotient = uii / 100;
*start++ = '0' + quotient;
quotient = uii - 100 * quotient;
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
if (!remainder) {
return start;
}
*start++ = '.';
quotient = remainder / 1000;
memcpy(start, &(digit2_table[quotient * 2]), 2);
remainder -= quotient * 1000;
if (!remainder) {
goto dtoa_so8_tail;
}
start += 2;
dtoa_so8_pretail3:
quotient = remainder / 10;
memcpy(start, &(digit2_table[quotient * 2]), 2);
remainder -= quotient * 10;
if (!remainder) {
goto dtoa_so8_tail;
}
start[2] = '0' + remainder;
return &(start[3]);
}
double_bround4(dxx, banker_round6, &uii, &remainder);
quotient = uii / 100;
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
quotient = uii - (100 * quotient);
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
if (!remainder) {
return start;
}
*start++ = '.';
goto dtoa_so8_pretail4;
} else if (dxx < 999999.99499999) {
if (dxx < 99999.999499999) {
double_bround3(dxx, banker_round6, &uii, &remainder);
quotient = uii / 10000;
*start = '0' + quotient;
uii -= 10000 * quotient;
quotient = uii / 100;
start = memcpya(&(start[1]), &(digit2_table[quotient * 2]), 2);
uii -= 100 * quotient;
start = memcpya(start, &(digit2_table[uii * 2]), 2);
if (!remainder) {
return start;
}
*start++ = '.';
goto dtoa_so8_pretail3;
}
double_bround2(dxx, banker_round6, &uii, &remainder);
quotient = uii / 10000;
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
uii -= 10000 * quotient;
quotient = uii / 100;
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
uii -= 100 * quotient;
start = memcpya(start, &(digit2_table[uii * 2]), 2);
if (!remainder) {
return start;
}
*start++ = '.';
goto dtoa_so8_pretail2;
} else if (dxx < 9999999.9499999) {
double_bround1(dxx, banker_round6, &uii, &remainder);
quotient = uii / 1000000;
*start = '0' + quotient;
uii -= 1000000 * quotient;
quotient = uii / 10000;
start = memcpya(&(start[1]), &(digit2_table[quotient * 2]), 2);
uii -= 10000 * quotient;
quotient = uii / 100;
start = memcpya(start, &(digit2_table[quotient * 2]), 2);
uii -= 100 * quotient;
start = memcpya(start, &(digit2_table[uii * 2]), 2);
if (!remainder) {
return start;
}
*start = '.';
start[1] = '0' + remainder;
return &(start[2]);
} else {
return uitoa_z8(double_bround(dxx, banker_round6), start);
}
}
char* dtoa_e(double dxx, char* start) {
uint32_t xp10 = 0;
uint32_t quotient;
uint32_t remainder;
char sign;
if (dxx != dxx) {
// do this first to avoid generating exception
return memcpyl3a(start, "nan");
} else if (dxx < 0) {
*start++ = '-';
dxx = -dxx;
}
if (dxx >= 9.9999994999999e-1) {
if (dxx >= 9.9999994999999e7) {
if (dxx >= 9.9999994999999e127) {
if (dxx == INFINITY) {
return memcpyl3a(start, "inf");
} else if (dxx >= 9.9999994999999e255) {
dxx *= 1.0e-256;
xp10 |= 256;
} else {
dxx *= 1.0e-128;
xp10 |= 128;
}
}
if (dxx >= 9.9999994999999e63) {
dxx *= 1.0e-64;
xp10 |= 64;
}
if (dxx >= 9.9999994999999e31) {
dxx *= 1.0e-32;
xp10 |= 32;
}
if (dxx >= 9.9999994999999e15) {
dxx *= 1.0e-16;
xp10 |= 16;
}
if (dxx >= 9.9999994999999e7) {
dxx *= 1.0e-8;
xp10 |= 8;
}
}
if (dxx >= 9.9999994999999e3) {
dxx *= 1.0e-4;
xp10 |= 4;
}
if (dxx >= 9.9999994999999e1) {
dxx *= 1.0e-2;
xp10 |= 2;
}
if (dxx >= 9.9999994999999) {
dxx *= 1.0e-1;
xp10++;
}
sign = '+';
} else {
if (dxx < 9.9999994999999e-8) {
// general case
if (dxx < 9.9999994999999e-128) {
if (dxx == 0.0) {
return memcpya(start, "0.000000e+00", 12);
}
if (dxx < 9.9999994999999e-256) {
dxx *= 1.0e256;
xp10 |= 256;
} else {
dxx *= 1.0e128;
xp10 |= 128;
}
}
if (dxx < 9.9999994999999e-64) {
dxx *= 1.0e64;
xp10 |= 64;
}
if (dxx < 9.9999994999999e-32) {
dxx *= 1.0e32;
xp10 |= 32;
}
if (dxx < 9.9999994999999e-16) {
dxx *= 1.0e16;
xp10 |= 16;
}
if (dxx < 9.9999994999999e-8) {
dxx *= 100000000;
xp10 |= 8;
}
}
if (dxx < 9.999994999999e-4) {
dxx *= 10000;
xp10 |= 4;
}
if (dxx < 9.9999994999999e-2) {
dxx *= 100;
xp10 |= 2;
}
if (dxx < 9.9999994999999e-1) {
dxx *= 10;
xp10++;
}
sign = '-';
}
double_bround6(dxx, banker_round7, "ient, &remainder);
*start++ = '0' + quotient;
*start++ = '.';
start = uitoa_z6(remainder, start);
*start++ = 'e';
*start++ = sign;
if (xp10 >= 100) {
quotient = xp10 / 100;
*start++ = '0' + quotient;
xp10 -= quotient * 100;
}
return memcpya(start, &(digit2_table[xp10 * 2]), 2);
}
char* ftoa_e(float fxx, char* start) {
uint32_t xp10 = 0;
uint32_t quotient;
uint32_t remainder;
char sign;
if (fxx != fxx) {
// do this first to avoid generating exception
return memcpyl3a(start, "nan");
} else if (fxx < 0) {
*start++ = '-';
fxx = -fxx;
}
if (fxx >= 9.9999995e-1) {
if (fxx >= 9.9999995e15) {
if (fxx == INFINITY) {
return memcpyl3a(start, "inf");
} else if (fxx >= 9.9999995e31) {
fxx *= 1.0e-32;
xp10 |= 32;
} else {
fxx *= 1.0e-16;
xp10 |= 16;
}
}
if (fxx >= 9.9999995e7) {
fxx *= 1.0e-8;
xp10 |= 8;
}
if (fxx >= 9.9999995e3) {
fxx *= 1.0e-4;
xp10 |= 4;
}
if (fxx >= 9.9999995e1) {
fxx *= 1.0e-2;
xp10 |= 2;
}
if (fxx >= 9.9999995) {
fxx *= 1.0e-1;
xp10++;
}
sign = '+';
} else {
if (fxx < 9.9999995e-16) {
if (fxx == 0.0) {
return memcpya(start, "0.000000e+00", 12);
} else if (fxx < 9.9999995e-32) {
fxx *= 1.0e32;
xp10 |= 32;
} else {
fxx *= 1.0e16;
xp10 |= 16;
}
}
if (fxx < 9.9999995e-8) {
fxx *= 100000000;
xp10 |= 8;
}
if (fxx < 9.9999995e-4) {
fxx *= 10000;
xp10 |= 4;
}
if (fxx < 9.9999995e-2) {
fxx *= 100;
xp10 |= 2;
}
if (fxx < 9.9999995e-1) {
fxx *= 10;
xp10++;
}
sign = '-';
}
float_round6(fxx, "ient, &remainder);
*start++ = '0' + quotient;
*start++ = '.';
start = uitoa_z6(remainder, start);
*start++ = 'e';
*start++ = sign;
return memcpya(start, &(digit2_table[xp10 * 2]), 2);
}
char* dtoa_f_p2(double dxx, char* start) {
const double* br_ptr;
uint32_t quotient;
uint32_t remainder;
if (dxx != dxx) {
return memcpyl3a(start, "nan");
} else if (dxx < 9.9949999999999) {
if (dxx < 0) {
*start++ = '-';
dxx = -dxx;
if (dxx >= 9.9949999999999) {
goto dtoa_f_p2_10;
}
}
double_bround2(dxx, banker_round11, "ient, &remainder);
*start++ = '0' + quotient;
dtoa_f_p2_dec:
*start++ = '.';
return memcpya(start, &(digit2_table[remainder * 2]), 2);
}
dtoa_f_p2_10:
if (dxx < 9999999.9949999) {
if (dxx < 999.99499999999) {
if (dxx < 99.994999999999) {
br_ptr = banker_round10;
} else {
br_ptr = banker_round9;
}
} else if (dxx < 99999.994999999) {
if (dxx < 9999.9949999999) {
br_ptr = banker_round8;
} else {
br_ptr = banker_round7;
}
} else if (dxx < 999999.99499999) {
br_ptr = banker_round6;
} else {
br_ptr = banker_round5;
}
double_bround2(dxx, br_ptr, "ient, &remainder);
start = uint32toa(quotient, start);
goto dtoa_f_p2_dec;
}
if (dxx == INFINITY) {
return memcpyl3a(start, "inf");
}
// just punt larger numbers to glibc for now, this isn't a bottleneck
start += sprintf(start, "%.2f", dxx);
return start;
}
char* dtoa_f_p3(double dxx, char* start) {
const double* br_ptr;
uint32_t quotient;
uint32_t remainder;
if (dxx != dxx) {
return memcpyl3a(start, "nan");
} else if (dxx < 9.9994999999999) {
if (dxx < 0) {
*start++ = '-';
dxx = -dxx;
if (dxx >= 9.9994999999999) {
goto dtoa_f_p3_10;
}
}
double_bround3(dxx, banker_round10, "ient, &remainder);
*start++ = '0' + quotient;
dtoa_f_p3_dec:
*start++ = '.';
quotient = remainder / 100;
remainder -= 100 * quotient;
*start++ = '0' + quotient;
return memcpya(start, &(digit2_table[remainder * 2]), 2);
}
dtoa_f_p3_10:
if (dxx < 999999.99949999) {
if (dxx < 999.99949999999) {
if (dxx < 99.999499999999) {
br_ptr = banker_round9;
} else {
br_ptr = banker_round8;
}
} else if (dxx < 99999.999499999) {
if (dxx < 9999.9994999999) {
br_ptr = banker_round7;
} else {
br_ptr = banker_round6;
}
} else {
br_ptr = banker_round5;
}
double_bround3(dxx, br_ptr, "ient, &remainder);
start = uint32toa(quotient, start);
goto dtoa_f_p3_dec;
}
if (dxx == INFINITY) {
return memcpyl3a(start, "inf");
}
start += sprintf(start, "%.3f", dxx);
return start;
}
char* dtoa_f_w9p6(double dxx, char* start) {
uint32_t quotient;
uint32_t remainder;
if (dxx != dxx) {
return memcpya(start, " nan", 9);
} else if (dxx < 9.9999994999999) {
if (dxx < 0) {
*start++ = '-';
dxx = -dxx;
if (dxx >= 9.9999994999999) {
goto dtoa_f_w9p6_10;
}
} else {
*start++ = ' ';
}
double_bround6(dxx, banker_round7, "ient, &remainder);
*start++ = '0' + quotient;
dtoa_f_w9p6_dec:
*start++ = '.';
return uitoa_z6(remainder, start);
}
dtoa_f_w9p6_10:
if (dxx < 999.99999949999) {
double_bround6(dxx, (dxx < 99.999999499999)? banker_round6 : banker_round5, "ient, &remainder);
start = uint32toa(quotient, start);
goto dtoa_f_w9p6_dec;
}
if (dxx == INFINITY) {
return memcpya(start, " inf", 9);
}
start += sprintf(start, "%.6f", dxx);
return start;
}
char* dtoa_f_w7p4(double dxx, char* start) {
const double* br_ptr;
uint32_t quotient;
uint32_t remainder;
if (dxx != dxx) {
return memcpya(start, " nan", 7);
} else if (dxx < 9.9999499999999) {
if (dxx < 0) {
*start++ = '-';
dxx = -dxx;
if (dxx >= 9.9999499999999) {
goto dtoa_f_w7p4_10;
}
} else {
*start++ = ' ';
}
double_bround4(dxx, banker_round9, "ient, &remainder);
*start++ = '0' + quotient;
dtoa_f_w7p4_dec:
*start++ = '.';
quotient = remainder / 100;
remainder -= 100 * quotient;
return memcpya(memcpya(start, &(digit2_table[quotient * 2]), 2), &(digit2_table[remainder * 2]), 2);
}
dtoa_f_w7p4_10:
if (dxx < 99999.999949999) {
if (dxx < 999.99994999999) {
if (dxx < 99.999949999999) {
br_ptr = banker_round8;
} else {
br_ptr = banker_round7;
}
} else if (dxx < 9999.9999499999) {
br_ptr = banker_round6;
} else {
br_ptr = banker_round5;
}
double_bround4(dxx, br_ptr, "ient, &remainder);
start = uint32toa(quotient, start);
goto dtoa_f_w7p4_dec;
}
if (dxx == INFINITY) {
return memcpya(start, " inf", 7);
}
start += sprintf(start, "%.4f", dxx);
return start;
}
char* dtoa_f_w9p6_spaced(double dxx, char* start) {
// Prettier fixed-width decimal: removes trailing zero(es) if and only if the
// match appears to be exact.
// Does not detect exact matches when abs(dxx) > 2^31 / 10^5.
double dyy = dxx * 100000 + 0.00000005;
start = dtoa_f_w9p6(dxx, start);
if (dyy - ((double)((int32_t)dyy)) >= 0.0000001) {
return start;
}
trailing_zeroes_to_spaces(start);
return start;
}
char* dtoa_f_w9p6_clipped(double dxx, char* start) {
// same conditions as _spaced()
double dyy = dxx * 100000 + 0.00000005;
start = dtoa_f_w9p6(dxx, start);
if (dyy - ((double)((int32_t)dyy)) >= 0.0000001) {
return start;
}
return clip_trailing_zeroes(start);
}
char* dtoa_g(double dxx, char* start) {
uint32_t xp10 = 0;
uint32_t quotient;
uint32_t remainder;
if (dxx != dxx) {
return memcpyl3a(start, "nan");
} else if (dxx < 0) {
*start++ = '-';
dxx = -dxx;
}
if (dxx < 9.9999949999999e-5) {
// 6 sig fig exponential notation, small
if (dxx < 9.9999949999999e-16) {
if (dxx < 9.9999949999999e-128) {
if (dxx == 0.0) {
*start = '0';
return &(start[1]);
} else if (dxx < 9.9999949999999e-256) {
dxx *= 1.0e256;
xp10 |= 256;
} else {
dxx *= 1.0e128;
xp10 |= 128;
}
}
if (dxx < 9.9999949999999e-64) {
dxx *= 1.0e64;
xp10 |= 64;
}
if (dxx < 9.9999949999999e-32) {
dxx *= 1.0e32;
xp10 |= 32;
}
if (dxx < 9.9999949999999e-16) {
dxx *= 1.0e16;
xp10 |= 16;
}
}
if (dxx < 9.9999949999999e-8) {
dxx *= 100000000;
xp10 |= 8;
}
if (dxx < 9.9999949999999e-4) {
dxx *= 10000;
xp10 |= 4;
}
if (dxx < 9.9999949999999e-2) {
dxx *= 100;
xp10 |= 2;
}
if (dxx < 9.9999949999999e-1) {
dxx *= 10;
xp10++;
}
double_bround5(dxx, banker_round8, "ient, &remainder);
start = memcpya(qrtoa_1p5(quotient, remainder, start), "e-", 2);
if (xp10 >= 100) {
quotient = xp10 / 100;
*start++ = '0' + quotient;
xp10 -= 100 * quotient;
}
return memcpya(start, &(digit2_table[xp10 * 2]), 2);
} else if (dxx >= 999999.49999999) {
// 6 sig fig exponential notation, large
if (dxx >= 9.9999949999999e15) {
if (dxx >= 9.9999949999999e127) {
if (dxx == INFINITY) {
return memcpyl3a(start, "inf");
} else if (dxx >= 9.9999949999999e255) {
dxx *= 1.0e-256;
xp10 |= 256;
} else {
dxx *= 1.0e-128;
xp10 |= 128;
}
}
if (dxx >= 9.9999949999999e63) {
dxx *= 1.0e-64;
xp10 |= 64;
}
if (dxx >= 9.9999949999999e31) {
dxx *= 1.0e-32;
xp10 |= 32;
}
if (dxx >= 9.9999949999999e15) {
dxx *= 1.0e-16;
xp10 |= 16;
}
}
if (dxx >= 9.9999949999999e7) {
dxx *= 1.0e-8;
xp10 |= 8;
}
if (dxx >= 9.9999949999999e3) {
dxx *= 1.0e-4;
xp10 |= 4;
}
if (dxx >= 9.9999949999999e1) {
dxx *= 1.0e-2;
xp10 |= 2;
}
if (dxx >= 9.9999949999999e0) {
dxx *= 1.0e-1;
xp10++;
}
double_bround5(dxx, banker_round8, "ient, &remainder);
start = memcpya(qrtoa_1p5(quotient, remainder, start), "e+", 2);
if (xp10 >= 100) {
quotient = xp10 / 100;
*start++ = '0' + quotient;
xp10 -= 100 * quotient;
}
return memcpya(start, &(digit2_table[xp10 * 2]), 2);
} else if (dxx >= 0.99999949999999) {
return dtoa_so6(dxx, start);
} else {
// 6 sig fig decimal, no less than ~0.0001
start = memcpya(start, "0.", 2);
if (dxx < 9.9999949999999e-3) {
dxx *= 100;
start = memcpya(start, "00", 2);
}
if (dxx < 9.9999949999999e-2) {
dxx *= 10;
*start++ = '0';
}
return uitoa_trunc6(double_bround(dxx * 1000000, banker_round8), start);
}
}
char* ftoa_g(float fxx, char* start) {
uint32_t xp10 = 0;
uint32_t quotient;
uint32_t remainder;
if (fxx != fxx) {
return memcpyl3a(start, "nan");
} else if (fxx < 0) {
*start++ = '-';
fxx = -fxx;
}
if (fxx < 9.9999944e-5) {
if (fxx < 9.9999944e-16) {
if (fxx == 0.0) {
*start = '0';
return &(start[1]);
} else if (fxx < 9.9999944e-32) {
fxx *= 1.0e32;
xp10 |= 32;
} else {
fxx *= 1.0e16;
xp10 |= 16;
}
}
if (fxx < 9.9999944e-8) {
fxx *= 100000000;
xp10 |= 8;
}
if (fxx < 9.9999944e-4) {
fxx *= 10000;
xp10 |= 4;
}
if (fxx < 9.9999944e-2) {
fxx *= 100;
xp10 |= 2;
}
if (fxx < 9.9999944e-1) {
fxx *= 10;
xp10++;
}
float_round5(fxx, "ient, &remainder);
return memcpya(memcpya(qrtoa_1p5(quotient, remainder, start), "e-", 2), &(digit2_table[xp10 * 2]), 2);
} else if (fxx >= 999999.44) {
if (fxx >= 9.9999944e15) {
if (fxx == INFINITY) {
return memcpyl3a(start, "inf");
} else if (fxx >= 9.9999944e31) {
fxx *= 1.0e-32;
xp10 |= 32;
} else {
fxx *= 1.0e-16;
xp10 |= 16;
}
}
if (fxx >= 9.9999944e7) {
fxx *= 1.0e-8;
xp10 |= 8;
}
if (fxx >= 9.9999944e3) {
fxx *= 1.0e-4;
xp10 |= 4;
}
if (fxx >= 9.9999944e1) {
fxx *= 1.0e-2;
xp10 |= 2;
}
if (fxx >= 9.9999944e0) {
fxx *= 1.0e-1;
xp10++;
}
float_round5(fxx, "ient, &remainder);
return memcpya(memcpya(qrtoa_1p5(quotient, remainder, start), "e+", 2), &(digit2_table[xp10 * 2]), 2);
} else if (fxx >= 0.99999944) {
return ftoa_so6(fxx, start);
} else {
// 6 sig fig decimal, no less than ~0.0001
start = memcpya(start, "0.", 2);
if (fxx < 9.9999944e-3) {
fxx *= 100;
start = memcpya(start, "00", 2);
}
if (fxx < 9.9999944e-2) {
fxx *= 10;
*start++ = '0';
}
return uitoa_trunc6(float_round(fxx * 1000000), start);
}
}
char* dtoa_g_wxp2(double dxx, uint32_t min_width, char* start) {
assert(min_width >= 5);
uint32_t xp10 = 0;
char wbuf[16];
char* wpos = wbuf;
uint32_t quotient;
uint32_t remainder;
if (dxx != dxx) {
memcpy(memseta(start, 32, min_width - 4), " nan", 4);
return &(start[min_width]);
} else if (dxx < 0) {
*wpos++ = '-';
dxx = -dxx;
}
if (dxx < 9.9499999999999e-5) {
// 2 sig fig exponential notation, small
if (dxx < 9.9499999999999e-16) {
if (dxx < 9.9499999999999e-128) {
if (dxx == 0.0) {
memset(start, 32, min_width - 1);
start[min_width - 1] = '0';
return &(start[min_width]);
} else if (dxx < 9.9499999999999e-256) {
dxx *= 1.0e256;
xp10 |= 256;
} else {
dxx *= 1.0e128;
xp10 |= 128;
}
}
if (dxx < 9.9499999999999e-64) {
dxx *= 1.0e64;
xp10 |= 64;
}
if (dxx < 9.9499999999999e-32) {
dxx *= 1.0e32;
xp10 |= 32;
}
if (dxx < 9.9499999999999e-16) {
dxx *= 1.0e16;
xp10 |= 16;
}
}
if (dxx < 9.9499999999999e-8) {
dxx *= 100000000;
xp10 |= 8;
}
if (dxx < 9.9499999999999e-4) {
dxx *= 10000;
xp10 |= 4;
}
if (dxx < 9.9499999999999e-2) {
dxx *= 100;
xp10 |= 2;
}
if (dxx < 9.9499999999999e-1) {
dxx *= 10;
xp10++;
}
double_bround1(dxx, banker_round12, "ient, &remainder);
wpos = qrtoa_1p1(quotient, remainder, wpos);
remainder = wpos - wbuf;
if (xp10 >= 100) {
if (remainder < min_width - 5) {
memcpy(memseta(start, 32, min_width - 5 - remainder), wbuf, remainder);
start = &(start[min_width - 5]);
} else {
start = memcpya(start, wbuf, remainder);
}
quotient = xp10 / 100;
start = memcpyax(start, "e-", 2, '0' + quotient);
xp10 -= 100 * quotient;
} else {
if (remainder < min_width - 4) {
memcpy(memseta(start, 32, min_width - 4 - remainder), wbuf, remainder);
start = &(start[min_width - 4]);
} else {
start = memcpya(start, wbuf, remainder);
}
start = memcpya(start, "e-", 2);
}
return memcpya(start, &(digit2_table[xp10 * 2]), 2);
} else if (dxx >= 99.499999999999) {
// 2 sig fig exponential notation, large
if (dxx >= 9.9499999999999e15) {
if (dxx >= 9.9499999999999e127) {
if (dxx == INFINITY) {
start = memseta(start, 32, min_width - 4);
if (wpos == wbuf) {
return memcpya(start, " inf", 4);
} else {
return memcpya(start, "-inf", 4);
}
} else if (dxx >= 9.9499999999999e255) {
dxx *= 1.0e-256;
xp10 |= 256;
} else {
dxx *= 1.0e-128;
xp10 |= 128;
}
}
if (dxx >= 9.9499999999999e63) {
dxx *= 1.0e-64;
xp10 |= 64;
}
if (dxx >= 9.9499999999999e31) {
dxx *= 1.0e-32;
xp10 |= 32;
}
if (dxx >= 9.9499999999999e15) {
dxx *= 1.0e-16;
xp10 |= 16;
}
}
if (dxx >= 9.9499999999999e7) {
dxx *= 1.0e-8;
xp10 |= 8;
}
if (dxx >= 9.9499999999999e3) {
dxx *= 1.0e-4;
xp10 |= 4;
}
if (dxx >= 9.9499999999999e1) {
dxx *= 1.0e-2;
xp10 |= 2;
}
if (dxx >= 9.9499999999999e0) {
dxx *= 1.0e-1;
xp10++;
}
double_bround1(dxx, banker_round12, "ient, &remainder);
wpos = qrtoa_1p1(quotient, remainder, wpos);
remainder = wpos - wbuf;
if (xp10 >= 100) {
if (remainder < min_width - 5) {
memcpy(memseta(start, 32, min_width - 5 - remainder), wbuf, remainder);
start = &(start[min_width - 5]);
} else {
start = memcpya(start, wbuf, remainder);
}
quotient = xp10 / 100;
start = memcpyax(start, "e+", 2, '0' + quotient);
xp10 -= 100 * quotient;
} else {
if (remainder < min_width - 4) {
memcpy(memseta(start, 32, min_width - 4 - remainder), wbuf, remainder);
start = &(start[min_width - 4]);
} else {
start = memcpya(start, wbuf, remainder);
}
start = memcpya(start, "e+", 2);
}
return memcpya(start, &(digit2_table[xp10 * 2]), 2);
} else {
if (dxx >= 0.99499999999999) {
wpos = dtoa_so2(dxx, wpos);
} else {
// 2 sig fig decimal, no less than ~0.0001
wpos = memcpya(wpos, "0.", 2);
if (dxx < 9.9499999999999e-3) {
dxx *= 100;
wpos = memcpya(wpos, "00", 2);
}
if (dxx < 9.9499999999999e-2) {
dxx *= 10;
*wpos++ = '0';
}
wpos = uitoa_trunc2(double_bround(dxx * 100, banker_round12), wpos);
}
remainder = wpos - wbuf;
if (remainder < min_width) {
memcpy(memseta(start, 32, min_width - remainder), wbuf, remainder);
return &(start[min_width]);
} else {
return memcpya(start, wbuf, remainder);
}
}
}
char* dtoa_g_wxp3(double dxx, uint32_t min_width, char* start) {
assert(min_width >= 5);
uint32_t xp10 = 0;
char wbuf[16];
char* wpos = wbuf;
uint32_t quotient;
uint32_t remainder;
if (dxx != dxx) {
memcpy(memseta(start, 32, min_width - 4), " nan", 4);
return &(start[min_width]);
} else if (dxx < 0) {
*wpos++ = '-';
dxx = -dxx;
}
if (dxx < 9.9949999999999e-5) {
// 3 sig fig exponential notation, small
if (dxx < 9.9949999999999e-16) {
if (dxx < 9.9949999999999e-128) {
if (dxx == 0.0) {
memset(start, 32, min_width - 1);
start[min_width - 1] = '0';
return &(start[min_width]);
} else if (dxx < 9.9949999999999e-256) {
dxx *= 1.0e256;
xp10 |= 256;
} else {
dxx *= 1.0e128;
xp10 |= 128;
}
}
if (dxx < 9.9949999999999e-64) {
dxx *= 1.0e64;
xp10 |= 64;
}
if (dxx < 9.9949999999999e-32) {
dxx *= 1.0e32;
xp10 |= 32;
}
if (dxx < 9.9949999999999e-16) {
dxx *= 1.0e16;
xp10 |= 16;
}
}
if (dxx < 9.9949999999999e-8) {
dxx *= 100000000;
xp10 |= 8;
}
if (dxx < 9.9949999999999e-4) {
dxx *= 10000;
xp10 |= 4;
}
if (dxx < 9.9949999999999e-2) {
dxx *= 100;
xp10 |= 2;
}
if (dxx < 9.9949999999999e-1) {
dxx *= 10;
xp10++;
}
double_bround2(dxx, banker_round11, "ient, &remainder);
wpos = qrtoa_1p2(quotient, remainder, wpos);
remainder = wpos - wbuf;
if (xp10 >= 100) {
if (remainder < min_width - 5) {
memcpy(memseta(start, 32, min_width - 5 - remainder), wbuf, remainder);
start = &(start[min_width - 5]);
} else {
start = memcpya(start, wbuf, remainder);
}
quotient = xp10 / 100;
start = memcpyax(start, "e-", 2, '0' + quotient);
xp10 -= 100 * quotient;
} else {
if (remainder < min_width - 4) {
memcpy(memseta(start, 32, min_width - 4 - remainder), wbuf, remainder);
start = &(start[min_width - 4]);
} else {
start = memcpya(start, wbuf, remainder);
}
start = memcpya(start, "e-", 2);
}
return memcpya(start, &(digit2_table[xp10 * 2]), 2);
} else if (dxx >= 999.49999999999) {
// 3 sig fig exponential notation, large
if (dxx >= 9.9949999999999e15) {
if (dxx >= 9.9949999999999e127) {
if (dxx == INFINITY) {
start = memseta(start, 32, min_width - 4);
if (wpos == wbuf) {
return memcpya(start, " inf", 4);
} else {
return memcpya(start, "-inf", 4);
}
} else if (dxx >= 9.9949999999999e255) {
dxx *= 1.0e-256;
xp10 |= 256;
} else {
dxx *= 1.0e-128;
xp10 |= 128;
}
}
if (dxx >= 9.9949999999999e63) {
dxx *= 1.0e-64;
xp10 |= 64;
}
if (dxx >= 9.9949999999999e31) {
dxx *= 1.0e-32;
xp10 |= 32;
}
if (dxx >= 9.9949999999999e15) {
dxx *= 1.0e-16;
xp10 |= 16;
}
}
if (dxx >= 9.9949999999999e7) {
dxx *= 1.0e-8;
xp10 |= 8;
}
if (dxx >= 9.9949999999999e3) {
dxx *= 1.0e-4;
xp10 |= 4;
}
if (dxx >= 9.9949999999999e1) {
dxx *= 1.0e-2;
xp10 |= 2;
}
if (dxx >= 9.9949999999999e0) {
dxx *= 1.0e-1;
xp10++;
}
double_bround2(dxx, banker_round11, "ient, &remainder);
wpos = qrtoa_1p2(quotient, remainder, wpos);
remainder = wpos - wbuf;
if (xp10 >= 100) {
if (remainder < min_width - 5) {
memcpy(memseta(start, 32, min_width - 5 - remainder), wbuf, remainder);
start = &(start[min_width - 5]);
} else {
start = memcpya(start, wbuf, remainder);
}
quotient = xp10 / 100;
start = memcpyax(start, "e+", 2, '0' + quotient);
xp10 -= 100 * quotient;
} else {
if (remainder < min_width - 4) {
memcpy(memseta(start, 32, min_width - 4 - remainder), wbuf, remainder);
start = &(start[min_width - 4]);
} else {
start = memcpya(start, wbuf, remainder);
}
start = memcpya(start, "e+", 2);
}
return memcpya(start, &(digit2_table[xp10 * 2]), 2);
} else {
if (dxx >= 0.99949999999999) {
wpos = dtoa_so3(dxx, wpos);
} else {
// 3 sig fig decimal, no less than ~0.001
wpos = memcpya(wpos, "0.", 2);
if (dxx < 9.9949999999999e-3) {
dxx *= 100;
wpos = memcpya(wpos, "00", 2);
}
if (dxx < 9.9949999999999e-2) {
dxx *= 10;
*wpos++ = '0';
}
wpos = uitoa_trunc3(double_bround(dxx * 1000, banker_round11), wpos);
}
remainder = wpos - wbuf;
if (remainder < min_width) {
memcpy(memseta(start, 32, min_width - remainder), wbuf, remainder);
return &(start[min_width]);
} else {
return memcpya(start, wbuf, remainder);
}
}
}
char* dtoa_g_wxp4(double dxx, uint32_t min_width, char* start) {
uint32_t xp10 = 0;
char wbuf[16];
char* wpos = wbuf;
uint32_t quotient;
uint32_t remainder;
if (dxx != dxx) {
if (min_width > 3) {
start = memseta(start, 32, min_width - 3);
}
return memcpyl3a(start, "nan");
} else if (dxx < 0) {
*wpos++ = '-';
dxx = -dxx;
}
if (dxx < 9.9994999999999e-5) {
// 4 sig fig exponential notation, small
if (dxx < 9.9994999999999e-16) {
if (dxx < 9.9994999999999e-128) {
if (dxx == 0.0) {
memset(start, 32, min_width - 1);
start[min_width - 1] = '0';
return &(start[min_width]);
} else if (dxx < 9.9994999999999e-256) {
dxx *= 1.0e256;
xp10 |= 256;
} else {
dxx *= 1.0e128;
xp10 |= 128;
}
}
if (dxx < 9.9994999999999e-64) {
dxx *= 1.0e64;
xp10 |= 64;
}
if (dxx < 9.9994999999999e-32) {
dxx *= 1.0e32;
xp10 |= 32;
}
if (dxx < 9.9994999999999e-16) {
dxx *= 1.0e16;
xp10 |= 16;
}
}
if (dxx < 9.9994999999999e-8) {
dxx *= 100000000;
xp10 |= 8;
}
if (dxx < 9.9994999999999e-4) {
dxx *= 10000;
xp10 |= 4;
}
if (dxx < 9.9994999999999e-2) {
dxx *= 100;
xp10 |= 2;
}
if (dxx < 9.9994999999999e-1) {
dxx *= 10;
xp10++;
}
double_bround3(dxx, banker_round10, "ient, &remainder);
wpos = qrtoa_1p3(quotient, remainder, wpos);
remainder = wpos - wbuf;
if (xp10 >= 100) {
if (remainder + 5 < min_width) {
memcpy(memseta(start, 32, min_width - (remainder + 5)), wbuf, remainder);
start = &(start[min_width - 5]);
} else {
start = memcpya(start, wbuf, remainder);
}
quotient = xp10 / 100;
start = memcpyax(start, "e-", 2, '0' + quotient);
xp10 -= 100 * quotient;
} else {
if (remainder + 4 < min_width) {
memcpy(memseta(start, 32, min_width - (remainder + 4)), wbuf, remainder);
start = &(start[min_width - 4]);
} else {
start = memcpya(start, wbuf, remainder);
}
start = memcpya(start, "e-", 2);
}
return memcpya(start, &(digit2_table[xp10 * 2]), 2);
} else if (dxx >= 9999.4999999999) {
// 4 sig fig exponential notation, large
if (dxx >= 9.9994999999999e15) {
if (dxx >= 9.9994999999999e127) {
if (dxx == INFINITY) {
if (min_width > 4) {
start = memseta(start, 32, min_width - 4);
}
if (wpos == wbuf) {
return memcpya(start, " inf", 4);
} else {
return memcpya(start, "-inf", 4);
}
} else if (dxx >= 9.9994999999999e255) {
dxx *= 1.0e-256;
xp10 |= 256;
} else {
dxx *= 1.0e-128;
xp10 |= 128;
}
}
if (dxx >= 9.9994999999999e63) {
dxx *= 1.0e-64;
xp10 |= 64;
}
if (dxx >= 9.9994999999999e31) {
dxx *= 1.0e-32;
xp10 |= 32;
}
if (dxx >= 9.9994999999999e15) {
dxx *= 1.0e-16;
xp10 |= 16;
}
}
if (dxx >= 9.9994999999999e7) {
dxx *= 1.0e-8;
xp10 |= 8;
}
if (dxx >= 9.9994999999999e3) {
dxx *= 1.0e-4;
xp10 |= 4;
}
if (dxx >= 9.9994999999999e1) {
dxx *= 1.0e-2;
xp10 |= 2;
}
if (dxx >= 9.9994999999999e0) {
dxx *= 1.0e-1;
xp10++;
}
double_bround3(dxx, banker_round10, "ient, &remainder);
wpos = qrtoa_1p3(quotient, remainder, wpos);
remainder = wpos - wbuf;
if (xp10 >= 100) {
if (remainder + 5 < min_width) {
memcpy(memseta(start, 32, min_width - (remainder + 5)), wbuf, remainder);
start = &(start[min_width - 5]);
} else {
start = memcpya(start, wbuf, remainder);
}
quotient = xp10 / 100;
start = memcpyax(start, "e+", 2, '0' + quotient);
xp10 -= 100 * quotient;
} else {
if (remainder + 4 < min_width) {
memcpy(memseta(start, 32, min_width - (remainder + 4)), wbuf, remainder);
start = &(start[min_width - 4]);
} else {
start = memcpya(start, wbuf, remainder);
}
start = memcpya(start, "e+", 2);
}
return memcpya(start, &(digit2_table[xp10 * 2]), 2);
} else {
if (dxx >= 0.99994999999999) {
wpos = dtoa_so4(dxx, wpos);
} else {
// 4 sig fig decimal, no less than ~0.0001
wpos = memcpya(wpos, "0.", 2);
if (dxx < 9.9994999999999e-3) {
dxx *= 100;
wpos = memcpya(wpos, "00", 2);
}
if (dxx < 9.9994999999999e-2) {
dxx *= 10;
*wpos++ = '0';
}
wpos = uitoa_trunc4(double_bround(dxx * 10000, banker_round10), wpos);
}
remainder = wpos - wbuf;
if (remainder < min_width) {
memcpy(memseta(start, 32, min_width - remainder), wbuf, remainder);
return &(start[min_width]);
} else {
return memcpya(start, wbuf, remainder);
}
}
}
char* dtoa_g_wxp8(double dxx, uint32_t min_width, char* start) {
uint32_t xp10 = 0;
char wbuf[16];
char* wpos = wbuf;
uint32_t quotient;
uint32_t remainder;
if (dxx != dxx) {
if (min_width > 3) {
start = memseta(start, 32, min_width - 3);
}
return memcpyl3a(start, "nan");
} else if (dxx < 0) {
*wpos++ = '-';
dxx = -dxx;
}
if (dxx < 9.9999999499999e-5) {
// 8 sig fig exponential notation, small
if (dxx < 9.9999999499999e-16) {
if (dxx < 9.9999999499999e-128) {
if (dxx == 0.0) {
memset(start, 32, min_width - 1);
start[min_width - 1] = '0';
return &(start[min_width]);
} else if (dxx < 9.9999999499999e-256) {
dxx *= 1.0e256;
xp10 |= 256;
} else {
dxx *= 1.0e128;
xp10 |= 128;
}
}
if (dxx < 9.9999999499999e-64) {
dxx *= 1.0e64;
xp10 |= 64;
}
if (dxx < 9.9999999499999e-32) {
dxx *= 1.0e32;
xp10 |= 32;
}
if (dxx < 9.9999999499999e-16) {
dxx *= 1.0e16;
xp10 |= 16;
}
}
if (dxx < 9.9999999499999e-8) {
dxx *= 100000000;
xp10 |= 8;
}
if (dxx < 9.9999999499999e-4) {
dxx *= 10000;
xp10 |= 4;
}
if (dxx < 9.9999999499999e-2) {
dxx *= 100;
xp10 |= 2;
}
if (dxx < 9.9999999499999e-1) {
dxx *= 10;
xp10++;
}
double_bround7(dxx, banker_round6, "ient, &remainder);
wpos = qrtoa_1p7(quotient, remainder, wpos);
remainder = wpos - wbuf;
if (xp10 >= 100) {
if (remainder + 5 < min_width) {
memcpy(memseta(start, 32, min_width - (remainder + 5)), wbuf, remainder);
start = &(start[min_width - 5]);
} else {
start = memcpya(start, wbuf, remainder);
}
quotient = xp10 / 100;
start = memcpyax(start, "e-", 2, '0' + quotient);
xp10 -= 100 * quotient;
} else {
if (remainder + 4 < min_width) {
memcpy(memseta(start, 32, min_width - (remainder + 4)), wbuf, remainder);
start = &(start[min_width - 4]);
} else {
start = memcpya(start, wbuf, remainder);
}
start = memcpya(start, "e-", 2);
}
return memcpya(start, &(digit2_table[xp10 * 2]), 2);
} else if (dxx >= 99999999.499999) {
// 8 sig fig exponential notation, large
if (dxx >= 9.9999999499999e15) {
if (dxx >= 9.9999999499999e127) {
if (dxx == INFINITY) {
if (min_width > 4) {
start = memseta(start, 32, min_width - 4);
}
if (wpos == wbuf) {
return memcpya(start, " inf", 4);
} else {
return memcpya(start, "-inf", 4);
}
} else if (dxx >= 9.9999999499999e255) {
dxx *= 1.0e-256;
xp10 |= 256;
} else {
dxx *= 1.0e-128;
xp10 |= 128;
}
}
if (dxx >= 9.9999999499999e63) {
dxx *= 1.0e-64;
xp10 |= 64;
}
if (dxx >= 9.9999999499999e31) {
dxx *= 1.0e-32;
xp10 |= 32;
}
if (dxx >= 9.9999999499999e15) {
dxx *= 1.0e-16;
xp10 |= 16;
}
}
if (dxx >= 9.9999999499999e7) {
dxx *= 1.0e-8;
xp10 |= 8;
}
if (dxx >= 9.9999999499999e3) {
dxx *= 1.0e-4;
xp10 |= 4;
}
if (dxx >= 9.9999999499999e1) {
dxx *= 1.0e-2;
xp10 |= 2;
}
if (dxx >= 9.9999999499999e0) {
dxx *= 1.0e-1;
xp10++;
}
double_bround7(dxx, banker_round6, "ient, &remainder);
wpos = qrtoa_1p7(quotient, remainder, wpos);
remainder = wpos - wbuf;
if (xp10 >= 100) {
if (remainder + 5 < min_width) {
memcpy(memseta(start, 32, min_width - (remainder + 5)), wbuf, remainder);
start = &(start[min_width - 5]);
} else {
start = memcpya(start, wbuf, remainder);
}
quotient = xp10 / 100;
start = memcpyax(start, "e+", 2, '0' + quotient);
xp10 -= 100 * quotient;
} else {
if (remainder + 4 < min_width) {
memcpy(memseta(start, 32, min_width - (remainder + 4)), wbuf, remainder);
start = &(start[min_width - 4]);
} else {
start = memcpya(start, wbuf, remainder);
}
start = memcpya(start, "e+", 2);
}
return memcpya(start, &(digit2_table[xp10 * 2]), 2);
} else {
if (dxx >= 0.99999999499999) {
wpos = dtoa_so8(dxx, wpos);
} else {
// 8 sig fig decimal, no less than ~0.0001
wpos = memcpya(wpos, "0.", 2);
if (dxx < 9.9999999499999e-3) {
dxx *= 100;
wpos = memcpya(wpos, "00", 2);
}
if (dxx < 9.9999999499999e-2) {
dxx *= 10;
*wpos++ = '0';
}
wpos = uitoa_trunc8(double_bround(dxx * 100000000, banker_round6), wpos);
}
remainder = wpos - wbuf;
if (remainder < min_width) {
memcpy(memseta(start, 32, min_width - remainder), wbuf, remainder);
return &(start[min_width]);
} else {
return memcpya(start, wbuf, remainder);
}
}
}
char* chrom_print_human(uint32_t num, char* buf) {
uint32_t n10;
if (num < 10) {
*buf = '0' + num;
return &(buf[1]);
} else if (num < 23) {
n10 = num / 10;
*buf = '0' + n10;
buf[1] = '0' + (num - 10 * n10);
return &(buf[2]);
} else if (num < 25) {
// X is 24th letter of alphabet, and 23rd chromosome
*buf = 'A' + num;
return &(buf[1]);
} else if (num > 26) {
// --allow-extra-chr 0
*buf = '0';
return &(buf[1]);
} else if (num == 25) {
memcpy(buf, "XY", 2);
return &(buf[2]);
} else {
memcpy(buf, "MT", 2);
return &(buf[2]);
}
}
void magic_num(uint32_t divisor, uint64_t* multp, uint32_t* __restrict pre_shiftp, uint32_t* __restrict post_shiftp, uint32_t* __restrict incrp) {
// Enables fast integer division by a constant not known until runtime. See
// http://ridiculousfish.com/blog/posts/labor-of-division-episode-iii.html .
// Assumes divisor is not zero, of course.
// May want to populate a struct instead.
uint32_t down_multiplier = 0;
uint32_t down_exponent = 0;
uint32_t has_magic_down = 0;
uint32_t quotient;
uint32_t remainder;
uint32_t ceil_log_2_d;
uint32_t exponent;
uint32_t uii;
if (divisor & (divisor - 1)) {
quotient = 0x80000000U / divisor;
remainder = 0x80000000U - (quotient * divisor);
ceil_log_2_d = 32 - __builtin_clz(divisor);
for (exponent = 0; ; exponent++) {
if (remainder >= divisor - remainder) {
quotient = quotient * 2 + 1;
remainder = remainder * 2 - divisor;
} else {
quotient = quotient * 2;
remainder = remainder * 2;
}
if ((exponent >= ceil_log_2_d) || (divisor - remainder) <= (1U << exponent)) {
break;
}
if ((!has_magic_down) && (remainder <= (1U << exponent))) {
has_magic_down = 1;
down_multiplier = quotient;
down_exponent = exponent;
}
}
if (exponent < ceil_log_2_d) {
*multp = quotient + 1;
*pre_shiftp = 0;
*post_shiftp = 32 + exponent;
*incrp = 0;
return;
} else if (divisor & 1) {
*multp = down_multiplier;
*pre_shiftp = 0;
*post_shiftp = 32 + down_exponent;
*incrp = 1;
return;
} else {
*pre_shiftp = __builtin_ctz(divisor);
magic_num(divisor >> (*pre_shiftp), multp, &uii, post_shiftp, incrp);
return;
}
} else {
// power of 2
*multp = 1;
*pre_shiftp = 0;
*post_shiftp = __builtin_ctz(divisor);
*incrp = 0;
}
}
void fill_bits(uintptr_t loc_start, uintptr_t len, uintptr_t* bitarr) {
assert(len);
uintptr_t maj_start = loc_start / BITCT;
uintptr_t maj_end = (loc_start + len) / BITCT;
uintptr_t minor;
if (maj_start == maj_end) {
bitarr[maj_start] |= (ONELU << ((loc_start + len) % BITCT)) - (ONELU << (loc_start % BITCT));
} else {
bitarr[maj_start] |= ~((ONELU << (loc_start % BITCT)) - ONELU);
fill_ulong_one(maj_end - maj_start - 1, &(bitarr[maj_start + 1]));
minor = (loc_start + len) % BITCT;
if (minor) {
bitarr[maj_end] |= (ONELU << minor) - ONELU;
}
}
}
void clear_bits(uintptr_t loc_start, uintptr_t len, uintptr_t* bitarr) {
assert(len);
uintptr_t maj_start = loc_start / BITCT;
uintptr_t maj_end = (loc_start + len) / BITCT;
uintptr_t minor;
if (maj_start == maj_end) {
bitarr[maj_start] &= ~((ONELU << ((loc_start + len) % BITCT)) - (ONELU << (loc_start % BITCT)));
} else {
bitarr[maj_start] &= ((ONELU << (loc_start % BITCT)) - ONELU);
fill_ulong_zero(maj_end - maj_start - 1, &(bitarr[maj_start + 1]));
minor = (loc_start + len) % BITCT;
if (minor) {
bitarr[maj_end] &= ~((ONELU << minor) - ONELU);
}
}
}
uint32_t next_unset_unsafe(const uintptr_t* bitarr, uint32_t loc) {
const uintptr_t* bitarr_ptr = &(bitarr[loc / BITCT]);
uintptr_t ulii = (~(*bitarr_ptr)) >> (loc % BITCT);
if (ulii) {
return loc + CTZLU(ulii);
}
do {
ulii = *(++bitarr_ptr);
} while (ulii == ~ZEROLU);
return ((uintptr_t)(bitarr_ptr - bitarr)) * BITCT + CTZLU(~ulii);
}
#ifdef __LP64__
uintptr_t next_unset_ul_unsafe(const uintptr_t* bitarr, uintptr_t loc) {
const uintptr_t* bitarr_ptr = &(bitarr[loc / BITCT]);
uintptr_t ulii = (~(*bitarr_ptr)) >> (loc % BITCT);
if (ulii) {
return loc + CTZLU(ulii);
}
do {
ulii = *(++bitarr_ptr);
} while (ulii == ~ZEROLU);
return (((uintptr_t)(bitarr_ptr - bitarr)) * BITCT + CTZLU(~ulii));
}
#endif
uint32_t next_unset(const uintptr_t* bitarr, uint32_t loc, uint32_t ceil) {
// safe version.
assert(ceil >= 1);
const uintptr_t* bitarr_ptr = &(bitarr[loc / BITCT]);
uintptr_t ulii = (~(*bitarr_ptr)) >> (loc % BITCT);
const uintptr_t* bitarr_last;
if (ulii) {
loc += CTZLU(ulii);
return MINV(loc, ceil);
}
bitarr_last = &(bitarr[(ceil - 1) / BITCT]);
do {
if (bitarr_ptr >= bitarr_last) {
return ceil;
}
ulii = *(++bitarr_ptr);
} while (ulii == ~ZEROLU);
loc = ((uintptr_t)(bitarr_ptr - bitarr)) * BITCT + CTZLU(~ulii);
return MINV(loc, ceil);
}
#ifdef __LP64__
uintptr_t next_unset_ul(const uintptr_t* bitarr, uintptr_t loc, uintptr_t ceil) {
const uintptr_t* bitarr_ptr = &(bitarr[loc / BITCT]);
uintptr_t ulii = (~(*bitarr_ptr)) >> (loc % BITCT);
const uintptr_t* bitarr_last;
if (ulii) {
ulii = loc + CTZLU(ulii);
return MINV(ulii, ceil);
}
bitarr_last = &(bitarr[(ceil - 1) / BITCT]);
do {
if (bitarr_ptr >= bitarr_last) {
return ceil;
}
ulii = *(++bitarr_ptr);
} while (ulii == ~ZEROLU);
ulii = ((uintptr_t)(bitarr_ptr - bitarr)) * BITCT + CTZLU(~ulii);
return MINV(ulii, ceil);
}
#endif
uint32_t next_set_unsafe(const uintptr_t* bitarr, uint32_t loc) {
const uintptr_t* bitarr_ptr = &(bitarr[loc / BITCT]);
uintptr_t ulii = (*bitarr_ptr) >> (loc % BITCT);
if (ulii) {
return loc + CTZLU(ulii);
}
do {
ulii = *(++bitarr_ptr);
} while (!ulii);
return ((uintptr_t)(bitarr_ptr - bitarr)) * BITCT + CTZLU(ulii);
}
#ifdef __LP64__
uintptr_t next_set_ul_unsafe(const uintptr_t* bitarr, uintptr_t loc) {
const uintptr_t* bitarr_ptr = &(bitarr[loc / BITCT]);
uintptr_t ulii = (*bitarr_ptr) >> (loc % BITCT);
if (ulii) {
return loc + CTZLU(ulii);
}
do {
ulii = *(++bitarr_ptr);
} while (!ulii);
return ((uintptr_t)(bitarr_ptr - bitarr)) * BITCT + CTZLU(ulii);
}
#endif
uint32_t next_set(const uintptr_t* bitarr, uint32_t loc, uint32_t ceil) {
const uintptr_t* bitarr_ptr = &(bitarr[loc / BITCT]);
uintptr_t ulii = (*bitarr_ptr) >> (loc % BITCT);
const uintptr_t* bitarr_last;
uint32_t rval;
if (ulii) {
rval = loc + CTZLU(ulii);
return MINV(rval, ceil);
}
bitarr_last = &(bitarr[(ceil - 1) / BITCT]);
do {
if (bitarr_ptr >= bitarr_last) {
return ceil;
}
ulii = *(++bitarr_ptr);
} while (!ulii);
rval = ((uintptr_t)(bitarr_ptr - bitarr)) * BITCT + CTZLU(ulii);
return MINV(rval, ceil);
}
#ifdef __LP64__
uintptr_t next_set_ul(const uintptr_t* bitarr, uintptr_t loc, uintptr_t ceil) {
const uintptr_t* bitarr_ptr = &(bitarr[loc / BITCT]);
uintptr_t ulii = (*bitarr_ptr) >> (loc % BITCT);
const uintptr_t* bitarr_last;
if (ulii) {
ulii = loc + CTZLU(ulii);
return MINV(ulii, ceil);
}
bitarr_last = &(bitarr[(ceil - 1) / BITCT]);
do {
if (bitarr_ptr >= bitarr_last) {
return ceil;
}
ulii = *(++bitarr_ptr);
} while (!ulii);
ulii = ((uintptr_t)(bitarr_ptr - bitarr)) * BITCT + CTZLU(ulii);
return MINV(ulii, ceil);
}
#endif
int32_t last_set_bit(const uintptr_t* bitarr, uint32_t word_ct) {
const uintptr_t* bitarr_ptr = &(bitarr[word_ct]);
uintptr_t ulii;
do {
ulii = *(--bitarr_ptr);
if (ulii) {
return ((uintptr_t)(bitarr_ptr - bitarr)) * BITCT + BITCT - 1 - CLZLU(ulii);
}
} while (bitarr_ptr > bitarr);
return -1;
}
int32_t last_clear_bit(const uintptr_t* bitarr, uint32_t ceil) {
// can return ceil or any lower number
const uintptr_t* bitarr_ptr = &(bitarr[ceil / BITCT]);
uint32_t remainder = ceil % BITCT;
uintptr_t ulii;
if (remainder) {
ulii = (~(*bitarr_ptr)) & ((ONELU << remainder) - ONELU);
if (ulii) {
return (ceil | (BITCT - 1)) - CLZLU(ulii);
}
}
while (bitarr_ptr > bitarr) {
ulii = ~(*(--bitarr_ptr));
if (ulii) {
return ((uintptr_t)(bitarr_ptr - bitarr)) * BITCT + BITCT - 1 - CLZLU(ulii);
}
}
return -1;
}
uint32_t prev_unset_unsafe(const uintptr_t* bitarr, uint32_t loc) {
// unlike the next_{un}set family, this always returns a STRICTLY earlier
// position
const uintptr_t* bitarr_ptr = &(bitarr[loc / BITCT]);
uint32_t remainder = loc % BITCT;
uintptr_t ulii;
if (remainder) {
ulii = (~(*bitarr_ptr)) & ((ONELU << remainder) - ONELU);
if (ulii) {
return (loc | (BITCT - 1)) - CLZLU(ulii);
}
}
do {
ulii = ~(*(--bitarr_ptr));
} while (!ulii);
return ((uintptr_t)(bitarr_ptr - bitarr)) * BITCT + BITCT - 1 - CLZLU(ulii);
}
/*
uint32_t prev_unset(uintptr_t* bitarr, uint32_t loc, uint32_t floor) {
uintptr_t* bitarr_ptr = &(bitarr[loc / BITCT]);
uint32_t remainder = loc % BITCT;
uintptr_t* bitarr_first;
uintptr_t ulii;
if (remainder) {
ulii = (~(*bitarr_ptr)) & ((ONELU << remainder) - ONELU);
if (ulii) {
loc = (loc | (BITCT - 1)) - CLZLU(ulii);
return MAXV(loc, floor);
}
}
bitarr_first = &(bitarr[floor / BITCT]);
do {
if (bitarr_ptr == bitarr_first) {
return floor;
}
ulii = ~(*(--bitarr_ptr));
} while (!ulii);
loc = ((uintptr_t)(bitarr_ptr - bitarr)) * BITCT + BITCT - 1 - CLZLU(ulii);
return MAXV(loc, floor);
}
*/
int32_t bigstack_calloc_uc(uintptr_t ct, unsigned char** ucp_ptr) {
*ucp_ptr = (unsigned char*)bigstack_alloc(ct);
if (!(*ucp_ptr)) {
return 1;
}
memset(*ucp_ptr, 0, ct);
return 0;
}
int32_t bigstack_calloc_d(uintptr_t ct, double** dp_ptr) {
*dp_ptr = (double*)bigstack_alloc(ct * sizeof(double));
if (!(*dp_ptr)) {
return 1;
}
fill_double_zero(ct, *dp_ptr);
return 0;
}
int32_t bigstack_calloc_f(uintptr_t ct, float** fp_ptr) {
*fp_ptr = (float*)bigstack_alloc(ct * sizeof(float));
if (!(*fp_ptr)) {
return 1;
}
fill_float_zero(ct, *fp_ptr);
return 0;
}
int32_t bigstack_calloc_ui(uintptr_t ct, uint32_t** uip_ptr) {
*uip_ptr = (uint32_t*)bigstack_alloc(ct * sizeof(int32_t));
if (!(*uip_ptr)) {
return 1;
}
fill_uint_zero(ct, *uip_ptr);
return 0;
}
int32_t bigstack_calloc_ul(uintptr_t ct, uintptr_t** ulp_ptr) {
*ulp_ptr = (uintptr_t*)bigstack_alloc(ct * sizeof(intptr_t));
if (!(*ulp_ptr)) {
return 1;
}
fill_ulong_zero(ct, *ulp_ptr);
return 0;
}
int32_t bigstack_calloc_ull(uintptr_t ct, uint64_t** ullp_ptr) {
*ullp_ptr = (uint64_t*)bigstack_alloc(ct * sizeof(int64_t));
if (!(*ullp_ptr)) {
return 1;
}
fill_ull_zero(ct, *ullp_ptr);
return 0;
}
int32_t bigstack_end_calloc_uc(uintptr_t ct, unsigned char** ucp_ptr) {
*ucp_ptr = (unsigned char*)bigstack_end_alloc(ct);
if (!(*ucp_ptr)) {
return 1;
}
memset(*ucp_ptr, 0, ct);
return 0;
}
int32_t bigstack_end_calloc_d(uintptr_t ct, double** dp_ptr) {
*dp_ptr = (double*)bigstack_end_alloc(ct * sizeof(double));
if (!(*dp_ptr)) {
return 1;
}
fill_double_zero(ct, *dp_ptr);
return 0;
}
int32_t bigstack_end_calloc_f(uintptr_t ct, float** fp_ptr) {
*fp_ptr = (float*)bigstack_end_alloc(ct * sizeof(float));
if (!(*fp_ptr)) {
return 1;
}
fill_float_zero(ct, *fp_ptr);
return 0;
}
int32_t bigstack_end_calloc_ui(uintptr_t ct, uint32_t** uip_ptr) {
*uip_ptr = (uint32_t*)bigstack_end_alloc(ct * sizeof(int32_t));
if (!(*uip_ptr)) {
return 1;
}
fill_uint_zero(ct, *uip_ptr);
return 0;
}
int32_t bigstack_end_calloc_ul(uintptr_t ct, uintptr_t** ulp_ptr) {
*ulp_ptr = (uintptr_t*)bigstack_end_alloc(ct * sizeof(intptr_t));
if (!(*ulp_ptr)) {
return 1;
}
fill_ulong_zero(ct, *ulp_ptr);
return 0;
}
int32_t bigstack_end_calloc_ull(uintptr_t ct, uint64_t** ullp_ptr) {
*ullp_ptr = (uint64_t*)bigstack_end_alloc(ct * sizeof(int64_t));
if (!(*ullp_ptr)) {
return 1;
}
fill_ull_zero(ct, *ullp_ptr);
return 0;
}
// MurmurHash3, from
// https://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp
static inline uint32_t rotl32(uint32_t x, int8_t r) {
return (x << r) | (x >> (32 - r));
}
static inline uint32_t getblock32(const uint32_t* p, int i) {
return p[i];
}
//-----------------------------------------------------------------------------
// Finalization mix - force all bits of a hash block to avalanche
static inline uint32_t fmix32(uint32_t h) {
h ^= h >> 16;
h *= 0x85ebca6b;
h ^= h >> 13;
h *= 0xc2b2ae35;
h ^= h >> 16;
return h;
}
uint32_t murmurhash3_32(const void* key, uint32_t len) {
const uint8_t* data = (const uint8_t*)key;
const int32_t nblocks = len / 4;
uint32_t h1 = 0;
// uint32_t h1 = seed;
const uint32_t c1 = 0xcc9e2d51;
const uint32_t c2 = 0x1b873593;
//----------
// body
const uint32_t* blocks = (const uint32_t*)(data + nblocks*4);
int32_t i;
uint32_t k1;
for(i = -nblocks; i; i++) {
k1 = getblock32(blocks,i);
k1 *= c1;
k1 = rotl32(k1,15);
k1 *= c2;
h1 ^= k1;
h1 = rotl32(h1,13);
h1 = h1*5+0xe6546b64;
}
//----------
// tail
const uint8_t* tail = (const uint8_t*)(data + nblocks*4);
k1 = 0;
switch(len & 3) {
case 3:
k1 ^= tail[2] << 16;
// fall through
case 2:
k1 ^= tail[1] << 8;
// fall through
case 1:
k1 ^= tail[0];
k1 *= c1;
k1 = rotl32(k1,15);
k1 *= c2;
h1 ^= k1;
}
//----------
// finalization
h1 ^= len;
return fmix32(h1);
}
uint32_t is_composite6(uintptr_t num) {
// assumes num is congruent to 1 or 5 mod 6.
// can speed this up by ~50% by hardcoding avoidance of multiples of 5/7,
// but this isn't currently a bottleneck so I'll keep this simple
uintptr_t divisor = 5;
while (divisor * divisor <= num) {
if (!(num % divisor)) {
return 1;
}
divisor += 2;
if (!(num % divisor)) {
return 1;
}
divisor += 4;
}
return 0;
}
uintptr_t geqprime(uintptr_t floor) {
// assumes floor is odd and greater than 1. Returns 5 if floor = 3,
// otherwise returns the first prime >= floor.
uintptr_t ulii = floor % 3;
if (!ulii) {
floor += 2;
} else if (ulii == 1) {
goto geqprime_1mod6;
}
while (is_composite6(floor)) {
floor += 2;
geqprime_1mod6:
if (!is_composite6(floor)) {
return floor;
}
floor += 4;
}
return floor;
}
int32_t populate_id_htable(uintptr_t unfiltered_ct, const uintptr_t* exclude_arr, uintptr_t item_ct, const char* item_ids, uintptr_t max_id_len, uint32_t store_dups, uint32_t id_htable_size, uint32_t* id_htable) {
// While unique IDs are normally assumed (and enforced) here, --extract and
// --exclude are an exception, since we want to be able to e.g. exclude all
// variants named '.'. Since there could be millions of them, ordinary
// O(n^2) hash table duplicate resolution is unacceptably slow; instead, we
// allocate additional linked lists past the end of id_htable to track all
// unfiltered indexes of duplicate names. (This requires the
// alloc_and_populate_id_htable interface; bigstack_end_alloc doesn't work
// there.)
uintptr_t item_uidx = 0;
uint32_t extra_alloc = 0;
uint32_t prev_llidx = 0;
// needs to be synced with extract_exclude_flag_norange()
uint32_t* extra_alloc_base = (uint32_t*)g_bigstack_base;
uint32_t item_idx = 0;
const char* sptr;
uintptr_t prev_uidx;
uintptr_t cur_bigstack_left;
uint32_t max_extra_alloc;
uint32_t slen;
uint32_t hashval;
uint32_t next_incr;
uint32_t top_diff;
uint32_t hash_result;
uint32_t cur_dup;
fill_uint_one(id_htable_size, id_htable);
if (!store_dups) {
for (; item_idx < item_ct; item_uidx++, item_idx++) {
next_unset_ul_unsafe_ck(exclude_arr, &item_uidx);
sptr = &(item_ids[item_uidx * max_id_len]);
slen = strlen(sptr);
hashval = murmurhash3_32(sptr, slen) % id_htable_size;
next_incr = 1;
while (1) {
hash_result = id_htable[hashval];
if (hash_result == 0xffffffffU) {
id_htable[hashval] = item_uidx;
break;
} else if (!memcmp(sptr, &(item_ids[hash_result * max_id_len]), slen + 1)) {
// could add an allow_dups parameter which controls whether this is
// an error
LOGERRPRINTFWW("Error: Duplicate ID '%s'.\n", sptr);
return RET_INVALID_FORMAT;
}
// defend against overflow
top_diff = id_htable_size - hashval;
if (top_diff > next_incr) {
hashval += next_incr;
} else {
hashval = next_incr - top_diff;
}
next_incr += 2; // quadratic probing
}
}
} else {
cur_bigstack_left = bigstack_left();
#ifdef __LP64__
if (cur_bigstack_left >= 0x400000000LLU) {
max_extra_alloc = 0xfffffffeU;
} else {
max_extra_alloc = cur_bigstack_left / sizeof(int32_t);
}
#else
max_extra_alloc = cur_bigstack_left / sizeof(int32_t);
#endif
for (; item_idx < item_ct; item_uidx++, item_idx++) {
next_unset_ul_unsafe_ck(exclude_arr, &item_uidx);
sptr = &(item_ids[item_uidx * max_id_len]);
slen = strlen(sptr);
hashval = murmurhash3_32(sptr, slen) % id_htable_size;
next_incr = 1;
while (1) {
hash_result = id_htable[hashval];
if (hash_result == 0xffffffffU) {
id_htable[hashval] = item_uidx;
break;
} else {
cur_dup = hash_result >> 31;
if (cur_dup) {
prev_llidx = hash_result << 1;
prev_uidx = extra_alloc_base[prev_llidx];
} else {
prev_uidx = hash_result;
}
if (!memcmp(sptr, &(item_ids[prev_uidx * max_id_len]), slen + 1)) {
if (extra_alloc + 4 > max_extra_alloc) {
return RET_NOMEM;
}
// point to linked list entry instead
if (!cur_dup) {
extra_alloc_base[extra_alloc] = hash_result;
extra_alloc_base[extra_alloc + 1] = 0xffffffffU; // list end
prev_llidx = extra_alloc;
extra_alloc += 2;
}
extra_alloc_base[extra_alloc] = item_uidx;
extra_alloc_base[extra_alloc + 1] = prev_llidx;
id_htable[hashval] = 0x80000000U | (extra_alloc >> 1);
extra_alloc += 2;
break; // bugfix
}
}
top_diff = id_htable_size - hashval;
if (top_diff > next_incr) {
hashval += next_incr;
} else {
hashval = next_incr - top_diff;
}
next_incr += 2;
}
}
if (extra_alloc) {
bigstack_alloc(extra_alloc * sizeof(int32_t));
}
}
return 0;
}
uint32_t id_htable_find(const char* id_buf, uintptr_t cur_id_len, const uint32_t* id_htable, uint32_t id_htable_size, const char* item_ids, uintptr_t max_id_len) {
// assumes no duplicate entries, and nonzero id_htable_size
// returns 0xffffffffU on failure
if (cur_id_len >= max_id_len) {
return 0xffffffffU;
}
uint32_t hashval = murmurhash3_32(id_buf, cur_id_len) % id_htable_size;
uint32_t next_incr = 1;
const char* sptr;
uint32_t hash_result;
uint32_t top_diff;
while (1) {
hash_result = id_htable[hashval];
if (hash_result == 0xffffffffU) {
return 0xffffffffU;
}
sptr = &(item_ids[hash_result * max_id_len]);
if ((!memcmp(id_buf, sptr, cur_id_len)) && (!sptr[cur_id_len])) {
return hash_result;
}
top_diff = id_htable_size - hashval;
if (top_diff > next_incr) {
hashval += next_incr;
} else {
hashval = next_incr - top_diff;
}
next_incr += 2;
}
}
void fill_idx_to_uidx(const uintptr_t* exclude_arr, uintptr_t unfiltered_item_ct, uintptr_t item_ct, uint32_t* idx_to_uidx) {
uint32_t* idx_to_uidx_end = &(idx_to_uidx[item_ct]);
uint32_t item_uidx = 0;
uint32_t item_uidx_stop;
while (idx_to_uidx < idx_to_uidx_end) {
item_uidx = next_unset_unsafe(exclude_arr, item_uidx);
item_uidx_stop = next_set(exclude_arr, item_uidx, unfiltered_item_ct);
do {
*idx_to_uidx++ = item_uidx++;
} while (item_uidx < item_uidx_stop);
}
}
void fill_idx_to_uidx_incl(const uintptr_t* include_arr, uintptr_t unfiltered_item_ct, uintptr_t item_ct, uint32_t* idx_to_uidx) {
uint32_t* idx_to_uidx_end = &(idx_to_uidx[item_ct]);
uint32_t item_uidx = 0;
uint32_t item_uidx_stop;
while (idx_to_uidx < idx_to_uidx_end) {
item_uidx = next_set_unsafe(include_arr, item_uidx);
item_uidx_stop = next_unset(include_arr, item_uidx, unfiltered_item_ct);
do {
*idx_to_uidx++ = item_uidx++;
} while (item_uidx < item_uidx_stop);
}
}
void fill_uidx_to_idx(const uintptr_t* exclude_arr, uint32_t unfiltered_item_ct, uint32_t item_ct, uint32_t* uidx_to_idx) {
uint32_t item_uidx = 0;
uint32_t item_idx = 0;
uint32_t* uidx_to_idx_ptr;
uint32_t* uidx_to_idx_stop;
while (item_idx < item_ct) {
item_uidx = next_unset_unsafe(exclude_arr, item_uidx);
uidx_to_idx_ptr = &(uidx_to_idx[item_uidx]);
item_uidx = next_set(exclude_arr, item_uidx, unfiltered_item_ct);
uidx_to_idx_stop = &(uidx_to_idx[item_uidx]);
do {
*uidx_to_idx_ptr++ = item_idx++;
} while (uidx_to_idx_ptr < uidx_to_idx_stop);
}
}
void fill_uidx_to_idx_incl(const uintptr_t* include_arr, uint32_t unfiltered_item_ct, uint32_t item_ct, uint32_t* uidx_to_idx) {
uint32_t item_uidx = 0;
uint32_t item_idx = 0;
uint32_t* uidx_to_idx_ptr;
uint32_t* uidx_to_idx_stop;
while (item_idx < item_ct) {
item_uidx = next_set_unsafe(include_arr, item_uidx);
uidx_to_idx_ptr = &(uidx_to_idx[item_uidx]);
item_uidx = next_unset(include_arr, item_uidx, unfiltered_item_ct);
uidx_to_idx_stop = &(uidx_to_idx[item_uidx]);
do {
*uidx_to_idx_ptr++ = item_idx++;
} while (uidx_to_idx_ptr < uidx_to_idx_stop);
}
}
void fill_midx_to_idx(const uintptr_t* exclude_arr_orig, const uintptr_t* exclude_arr, uint32_t item_ct, uint32_t* midx_to_idx) {
// Assumes item_ct is nonzero.
// May want to switch to alternate behavior: when current midx is excluded,
// fill midx_to_idx[] with the next item_idx.
uint32_t item_uidx = next_unset_unsafe(exclude_arr_orig, 0);
uint32_t item_idx = 0;
uint32_t item_midx;
for (item_midx = 0; item_idx < item_ct; item_uidx++, item_midx++) {
next_unset_unsafe_ck(exclude_arr_orig, &item_uidx);
if (!IS_SET(exclude_arr, item_uidx)) {
midx_to_idx[item_midx] = item_idx++;
}
}
}
void fill_quatervec_55(uint32_t ct, uintptr_t* quatervec) {
uint32_t rem = ct & (BITCT - 1);
#ifdef __LP64__
const __m128i m1 = {FIVEMASK, FIVEMASK};
__m128i* vecp = (__m128i*)quatervec;
__m128i* vec_end = (__m128i*)(&(quatervec[2 * (ct / BITCT)]));
uintptr_t* second_to_last;
while (vecp < vec_end) {
*vecp++ = m1;
}
if (rem) {
second_to_last = (uintptr_t*)vecp;
if (rem > BITCT2) {
second_to_last[0] = FIVEMASK;
second_to_last[1] = FIVEMASK >> ((BITCT - rem) * 2);
} else {
second_to_last[0] = FIVEMASK >> ((BITCT2 - rem) * 2);
second_to_last[1] = 0;
}
}
#else
uintptr_t* vec_end = &(quatervec[2 * (ct / BITCT)]);
while (quatervec < vec_end) {
*quatervec++ = FIVEMASK;
}
if (rem) {
if (rem > BITCT2) {
quatervec[0] = FIVEMASK;
quatervec[1] = FIVEMASK >> ((BITCT - rem) * 2);
} else {
quatervec[0] = FIVEMASK >> ((BITCT2 - rem) * 2);
quatervec[1] = 0;
}
}
#endif
}
void quaterarr_collapse_init(const uintptr_t* __restrict unfiltered_bitarr, uint32_t unfiltered_ct, const uintptr_t* __restrict filter_bitarr, uint32_t filtered_ct, uintptr_t* __restrict output_quaterarr) {
// Used to unpack e.g. unfiltered sex_male to a filtered quaterarr usable as
// a raw input bitmask.
// Assumes output_quaterarr is sized to a multiple of 16 bytes.
uintptr_t cur_write = 0;
uint32_t item_uidx = 0;
uint32_t write_bit = 0;
uint32_t item_idx = 0;
uint32_t item_uidx_stop;
while (item_idx < filtered_ct) {
item_uidx = next_set_unsafe(filter_bitarr, item_uidx);
item_uidx_stop = next_unset(filter_bitarr, item_uidx, unfiltered_ct);
item_idx += item_uidx_stop - item_uidx;
do {
cur_write |= ((unfiltered_bitarr[item_uidx / BITCT] >> (item_uidx % BITCT)) & 1) << (write_bit * 2);
if (++write_bit == BITCT2) {
*output_quaterarr++ = cur_write;
cur_write = 0;
write_bit = 0;
}
} while (++item_uidx < item_uidx_stop);
}
if (write_bit) {
*output_quaterarr++ = cur_write;
}
if ((filtered_ct + (BITCT2 - 1)) & BITCT2) {
*output_quaterarr = 0;
}
}
void quaterarr_collapse_init_exclude(const uintptr_t* __restrict unfiltered_bitarr, uint32_t unfiltered_ct, const uintptr_t* __restrict filter_exclude_bitarr, uint32_t filtered_ct, uintptr_t* __restrict output_quaterarr) {
uintptr_t cur_write = 0;
uint32_t item_uidx = 0;
uint32_t write_bit = 0;
uint32_t item_idx = 0;
uint32_t item_uidx_stop;
while (item_idx < filtered_ct) {
item_uidx = next_unset_unsafe(filter_exclude_bitarr, item_uidx);
item_uidx_stop = next_set(filter_exclude_bitarr, item_uidx, unfiltered_ct);
item_idx += item_uidx_stop - item_uidx;
do {
cur_write |= ((unfiltered_bitarr[item_uidx / BITCT] >> (item_uidx % BITCT)) & 1) << (write_bit * 2);
if (++write_bit == BITCT2) {
*output_quaterarr++ = cur_write;
cur_write = 0;
write_bit = 0;
}
} while (++item_uidx < item_uidx_stop);
}
if (write_bit) {
*output_quaterarr++ = cur_write;
}
if ((filtered_ct + (BITCT2 - 1)) & BITCT2) {
*output_quaterarr = 0;
}
}
uint32_t alloc_collapsed_haploid_filters(const uintptr_t* __restrict sample_bitarr, const uintptr_t* __restrict sex_male, uint32_t unfiltered_sample_ct, uint32_t sample_ct, uint32_t hh_exists, uint32_t is_include, uintptr_t** sample_include_quatervec_ptr, uintptr_t** sample_male_include_quatervec_ptr) {
uintptr_t sample_ctv2 = QUATERCT_TO_ALIGNED_WORDCT(sample_ct);
if (hh_exists & (Y_FIX_NEEDED | NXMHH_EXISTS)) {
// if already allocated, we assume this is fully initialized
if (!(*sample_include_quatervec_ptr)) {
if (bigstack_alloc_ul(sample_ctv2, sample_include_quatervec_ptr)) {
return 1;
}
fill_quatervec_55(sample_ct, *sample_include_quatervec_ptr);
}
}
if (hh_exists & (XMHH_EXISTS | Y_FIX_NEEDED)) {
// if already allocated, we assume it's been bigstack_end_alloc'd but not
// initialized
if (!(*sample_male_include_quatervec_ptr)) {
if (bigstack_alloc_ul(sample_ctv2, sample_male_include_quatervec_ptr)) {
return 1;
}
}
if (is_include) {
quaterarr_collapse_init(sex_male, unfiltered_sample_ct, sample_bitarr, sample_ct, *sample_male_include_quatervec_ptr);
} else {
quaterarr_collapse_init_exclude(sex_male, unfiltered_sample_ct, sample_bitarr, sample_ct, *sample_male_include_quatervec_ptr);
}
}
return 0;
}
void sample_delim_convert(uintptr_t unfiltered_sample_ct, const uintptr_t* sample_exclude, uint32_t sample_ct, uintptr_t max_sample_id_len, char oldc, char newc, char* sample_ids) {
// assumes there is exactly one delimiter to convert per name
uintptr_t sample_uidx = 0;
uint32_t sample_idx;
char* nptr;
for (sample_idx = 0; sample_idx < sample_ct; sample_uidx++, sample_idx++) {
next_unset_ul_unsafe_ck(sample_exclude, &sample_uidx);
nptr = (char*)memchr(&(sample_ids[sample_uidx * max_sample_id_len]), (unsigned char)oldc, max_sample_id_len);
*nptr = newc;
}
}
void get_set_wrange_align(const uintptr_t* __restrict bitarr, uintptr_t word_ct, uintptr_t* __restrict firstw_ptr, uintptr_t* __restrict wlen_ptr) {
const uintptr_t* bitarr_ptr = bitarr;
const uintptr_t* bitarr_end = &(bitarr[word_ct]);
#ifdef __LP64__
const uintptr_t* bitarr_end2 = &(bitarr[word_ct & (~ONELU)]);
while (bitarr_ptr < bitarr_end2) {
if (bitarr_ptr[0] || bitarr_ptr[1]) {
*firstw_ptr = (uintptr_t)(bitarr_ptr - bitarr);
while (!(*(--bitarr_end)));
*wlen_ptr = 1 + (uintptr_t)(bitarr_end - bitarr_ptr);
return;
}
bitarr_ptr = &(bitarr_ptr[2]);
}
if ((bitarr_end2 != bitarr_end) && (*bitarr_end2)) {
*firstw_ptr = word_ct - 1;
*wlen_ptr = 1;
return;
}
#else
while (bitarr_ptr < bitarr_end) {
if (*bitarr_ptr) {
*firstw_ptr = (uintptr_t)(bitarr_ptr - bitarr);
while (!(*(--bitarr_end)));
*wlen_ptr = 1 + (uintptr_t)(bitarr_end - bitarr_ptr);
return;
}
bitarr_ptr++;
}
#endif
*firstw_ptr = 0;
*wlen_ptr = 0;
}
// hashval computation left to caller since this is frequently used with
// chromosome IDs, where the compiler can optimize the integer modulus
// operation since the hash table size is preset
uint32_t unklen_id_htable_find(const char* cur_id, const char* const* item_ids, const uint32_t* id_htable, uint32_t hashval, uint32_t id_htable_size) {
// returns 0xffffffffU on failure
uint32_t next_incr = 1;
while (1) {
const uint32_t hash_result = id_htable[hashval];
if (hash_result == 0xffffffffU) {
return 0xffffffffU;
}
const char* htable_entry = item_ids[hash_result];
if (!strcmp(cur_id, htable_entry)) {
return hash_result;
}
const uint32_t top_diff = id_htable_size - hashval;
if (top_diff > next_incr) {
hashval += next_incr;
} else {
hashval = next_incr - top_diff;
}
next_incr += 2;
}
}
static inline uint32_t nonstd_chrom_name_htable_find(const char* chrom_name, const char* const* nonstd_names, const uint32_t* nonstd_id_htable, uint32_t name_slen) {
const uint32_t hashval = murmurhash3_32(chrom_name, name_slen) % CHROM_NAME_HTABLE_SIZE;
return unklen_id_htable_find(chrom_name, nonstd_names, nonstd_id_htable, hashval, CHROM_NAME_HTABLE_SIZE);
}
// Global since species_str() may be called by functions which don't actually
// care about chrom_info. (chrom_info is really a global variable too, but I
// find it easier to maintain this code when chrom_info dependencies are made
// explicit in the function signatures; in contrast, g_species_singular and
// g_species_plural are just for pretty printing and lend no insight into what
// the functions which reference them are doing.)
const char* g_species_singular = nullptr;
const char* g_species_plural = nullptr;
int32_t init_chrom_info(Chrom_info* chrom_info_ptr) {
// "constructor". initializes with maximum capacity. doesn't use bigstack.
// chrom_mask, haploid_mask: bits
// chrom_file_order, chrom_idx_to_foidx: int32s
// chrom_fo_vidx_start: int32s, with an extra trailing element
// nonstd_names: intptr_ts
// nonstd_id_htable: CHROM_NAME_HTABLE_SIZE int32s
assert(!(MAX_POSSIBLE_CHROM % VEC_BYTES));
const uintptr_t vecs_required = 2 * BITCT_TO_VECCT(MAX_POSSIBLE_CHROM) + 3 * (MAX_POSSIBLE_CHROM / VEC_INT32) + 1 + (MAX_POSSIBLE_CHROM / VEC_WORDS) + (CHROM_NAME_HTABLE_SIZE + (VEC_INT32 - 1)) / VEC_INT32;
// needed for proper cleanup
chrom_info_ptr->name_ct = 0;
chrom_info_ptr->incl_excl_name_stack = nullptr;
if (aligned_malloc(vecs_required * VEC_BYTES, &(chrom_info_ptr->chrom_mask))) {
return RET_NOMEM;
}
uintptr_t* alloc_iter = &(chrom_info_ptr->chrom_mask[BITCT_TO_VECCT(MAX_POSSIBLE_CHROM) * VEC_WORDS]);
chrom_info_ptr->haploid_mask = alloc_iter;
alloc_iter = &(alloc_iter[BITCT_TO_VECCT(MAX_POSSIBLE_CHROM) * VEC_WORDS]);
chrom_info_ptr->chrom_file_order = (uint32_t*)alloc_iter;
alloc_iter = &(alloc_iter[(MAX_POSSIBLE_CHROM / VEC_INT32) * VEC_WORDS]);
chrom_info_ptr->chrom_fo_vidx_start = (uint32_t*)alloc_iter;
alloc_iter = &(alloc_iter[((MAX_POSSIBLE_CHROM / VEC_INT32) + 1) * VEC_WORDS]);
chrom_info_ptr->chrom_idx_to_foidx = (uint32_t*)alloc_iter;
alloc_iter = &(alloc_iter[(MAX_POSSIBLE_CHROM / VEC_INT32) * VEC_WORDS]);
chrom_info_ptr->nonstd_names = (char**)alloc_iter;
alloc_iter = &(alloc_iter[MAX_POSSIBLE_CHROM]);
chrom_info_ptr->nonstd_id_htable = (uint32_t*)alloc_iter;
// alloc_iter = &(alloc_iter[((CHROM_NAME_HTABLE_SIZE + (VEC_INT32 - 1)) / VEC_INT32) * VEC_WORDS]);
// postpone nonstd_id_htable initialization until first nonstandard ID is
// loaded
// fill_uint_one(CHROM_NAME_HTABLE_SIZE, chrom_info_ptr->nonstd_id_htable);
return 0;
}
// if these are defined within init_species(), they may not persist after
// function exit
static const char species_singular_constants[][7] = {"person", "cow", "dog", "horse", "mouse", "plant", "sheep", "sample"};
static const char species_plural_constants[][8] = {"people", "cattle", "dogs", "horses", "mice", "plants", "sheep", "samples"};
void init_species(uint32_t species_code, Chrom_info* chrom_info_ptr) {
// human: 22, X, Y, XY, MT
// cow: 29, X, Y, MT
// dog: 38, X, Y, XY, MT
// horse: 31, X, Y
// mouse: 19, X, Y
// rice: 12
// sheep: 26, X, Y
const int32_t species_xymt_codes[] = {
23, 24, 25, 26,
30, 31, -2, 33,
39, 40, 41, 42,
32, 33, -2, -2,
20, 21, -2, -2,
-2, -2, -2, -2,
27, 28, -2, -2};
const uint32_t species_autosome_ct[] = {22, 29, 38, 31, 19, 12, 26};
const uint32_t species_max_code[] = {26, 33, 42, 33, 21, 12, 28};
fill_ulong_zero(CHROM_MASK_WORDS, chrom_info_ptr->chrom_mask);
chrom_info_ptr->output_encoding = 0;
chrom_info_ptr->zero_extra_chroms = 0;
chrom_info_ptr->species = species_code;
chrom_info_ptr->is_include_stack = 0;
g_species_singular = species_singular_constants[species_code];
g_species_plural = species_plural_constants[species_code];
if (species_code != SPECIES_UNKNOWN) {
// these are assumed to be already initialized in the SPECIES_UNKNOWN case
// bugfix: haploid_mask was being cleared in --chr-set case
fill_ulong_zero(CHROM_MASK_WORDS, chrom_info_ptr->haploid_mask);
memcpy(chrom_info_ptr->xymt_codes, &(species_xymt_codes[species_code * XYMT_OFFSET_CT]), XYMT_OFFSET_CT * sizeof(int32_t));
chrom_info_ptr->autosome_ct = species_autosome_ct[species_code];
chrom_info_ptr->max_code = species_max_code[species_code];
switch (species_code) {
case SPECIES_HUMAN:
chrom_info_ptr->haploid_mask[0] = 0x1800000;
break;
case SPECIES_COW:
chrom_info_ptr->haploid_mask[0] = 0xc0000000LU;
break;
case SPECIES_DOG:
#ifdef __LP64__
chrom_info_ptr->haploid_mask[0] = 0x18000000000LLU;
#else
chrom_info_ptr->haploid_mask[1] = 0x180;
#endif
break;
case SPECIES_HORSE:
#ifdef __LP64__
chrom_info_ptr->haploid_mask[0] = 0x300000000LLU;
#else
chrom_info_ptr->haploid_mask[1] = 3;
#endif
break;
case SPECIES_MOUSE:
chrom_info_ptr->haploid_mask[0] = 0x300000;
break;
case SPECIES_RICE:
chrom_info_ptr->haploid_mask[0] = 0x1fff;
break;
case SPECIES_SHEEP:
chrom_info_ptr->haploid_mask[0] = 0x18000000;
break;
}
}
fill_uint_one(chrom_info_ptr->max_code + 1, chrom_info_ptr->chrom_idx_to_foidx);
}
void init_default_chrom_mask(Chrom_info* chrom_info_ptr) {
if (chrom_info_ptr->species != SPECIES_UNKNOWN) {
fill_all_bits(chrom_info_ptr->max_code + 1, chrom_info_ptr->chrom_mask);
} else {
fill_all_bits(chrom_info_ptr->autosome_ct + 1, chrom_info_ptr->chrom_mask);
// --chr-set support
for (uint32_t xymt_idx = 0; xymt_idx < XYMT_OFFSET_CT; ++xymt_idx) {
int32_t cur_code = chrom_info_ptr->xymt_codes[xymt_idx];
if (cur_code != -2) {
set_bit(chrom_info_ptr->xymt_codes[xymt_idx], chrom_info_ptr->chrom_mask);
}
}
}
}
void forget_extra_chrom_names(uint32_t reinitialize, Chrom_info* chrom_info_ptr) {
const uint32_t name_ct = chrom_info_ptr->name_ct;
// guard against init_species() not being called yet
if (name_ct) {
char** nonstd_names = chrom_info_ptr->nonstd_names;
const uint32_t chrom_idx_last = chrom_info_ptr->max_code + name_ct;
for (uint32_t chrom_idx = chrom_info_ptr->max_code + 1; chrom_idx <= chrom_idx_last; ++chrom_idx) {
free(nonstd_names[chrom_idx]);
nonstd_names[chrom_idx] = nullptr;
}
if (reinitialize) {
fill_uint_one(CHROM_NAME_HTABLE_SIZE, chrom_info_ptr->nonstd_id_htable);
chrom_info_ptr->name_ct = 0;
}
}
}
int32_t finalize_chrom_info(Chrom_info* chrom_info_ptr) {
const uint32_t chrom_ct = chrom_info_ptr->chrom_ct;
const uint32_t name_ct = chrom_info_ptr->name_ct;
const uint32_t chrom_code_end = chrom_info_ptr->max_code + 1 + name_ct;
const uint32_t chrom_code_bitvec_ct = BITCT_TO_VECCT(chrom_code_end);
const uint32_t chrom_ct_int32vec_ct = (chrom_ct + (VEC_INT32 - 1)) / VEC_INT32;
const uint32_t chrom_ct_p1_int32vec_ct = 1 + (chrom_ct / VEC_INT32);
const uint32_t chrom_code_end_int32vec_ct = (chrom_code_end + (VEC_INT32 - 1)) / VEC_INT32;
const uint32_t chrom_code_end_wordvec_ct = (chrom_code_end + (VEC_WORDS - 1)) / VEC_WORDS;
uint32_t final_vecs_required = 2 * chrom_code_bitvec_ct + chrom_ct_int32vec_ct + chrom_ct_p1_int32vec_ct + chrom_code_end_int32vec_ct;
if (name_ct) {
final_vecs_required += chrom_code_end_wordvec_ct + (CHROM_NAME_HTABLE_SIZE + (VEC_INT32 - 1)) / VEC_INT32;
}
uintptr_t* new_alloc;
if (aligned_malloc(final_vecs_required * VEC_BYTES, &new_alloc)) {
return RET_NOMEM;
}
uintptr_t* old_alloc = chrom_info_ptr->chrom_mask;
uintptr_t* new_alloc_iter = new_alloc;
memcpy(new_alloc_iter, chrom_info_ptr->chrom_mask, chrom_code_bitvec_ct * VEC_BYTES);
chrom_info_ptr->chrom_mask = new_alloc_iter;
new_alloc_iter = &(new_alloc_iter[chrom_code_bitvec_ct * VEC_WORDS]);
memcpy(new_alloc_iter, chrom_info_ptr->haploid_mask, chrom_code_bitvec_ct * VEC_BYTES);
chrom_info_ptr->haploid_mask = new_alloc_iter;
new_alloc_iter = &(new_alloc_iter[chrom_code_bitvec_ct * VEC_WORDS]);
memcpy(new_alloc_iter, chrom_info_ptr->chrom_file_order, chrom_ct_int32vec_ct * VEC_BYTES);
chrom_info_ptr->chrom_file_order = (uint32_t*)new_alloc_iter;
new_alloc_iter = &(new_alloc_iter[chrom_ct_int32vec_ct * VEC_WORDS]);
memcpy(new_alloc_iter, chrom_info_ptr->chrom_fo_vidx_start, chrom_ct_p1_int32vec_ct * VEC_BYTES);
chrom_info_ptr->chrom_fo_vidx_start = (uint32_t*)new_alloc_iter;
new_alloc_iter = &(new_alloc_iter[chrom_ct_p1_int32vec_ct * VEC_WORDS]);
memcpy(new_alloc_iter, chrom_info_ptr->chrom_idx_to_foidx, chrom_code_end_int32vec_ct * VEC_BYTES);
chrom_info_ptr->chrom_idx_to_foidx = (uint32_t*)new_alloc_iter;
if (!name_ct) {
chrom_info_ptr->nonstd_names = nullptr;
chrom_info_ptr->nonstd_id_htable = nullptr;
} else {
new_alloc_iter = &(new_alloc_iter[chrom_code_end_int32vec_ct * VEC_WORDS]);
memcpy(new_alloc_iter, chrom_info_ptr->nonstd_names, chrom_code_end_wordvec_ct * VEC_BYTES);
chrom_info_ptr->nonstd_names = (char**)new_alloc_iter;
new_alloc_iter = &(new_alloc_iter[chrom_code_end_wordvec_ct * VEC_WORDS]);
memcpy(new_alloc_iter, chrom_info_ptr->nonstd_id_htable, CHROM_NAME_HTABLE_SIZE * sizeof(int32_t));
chrom_info_ptr->nonstd_id_htable = (uint32_t*)new_alloc_iter;
}
aligned_free(old_alloc);
return 0;
}
void cleanup_chrom_info(Chrom_info* chrom_info_ptr) {
if (chrom_info_ptr->chrom_mask) {
// bugfix: this must happened before aligned_free() call
forget_extra_chrom_names(0, chrom_info_ptr);
aligned_free(chrom_info_ptr->chrom_mask);
chrom_info_ptr->chrom_mask = nullptr;
}
Ll_str* ll_str_ptr = chrom_info_ptr->incl_excl_name_stack;
while (ll_str_ptr) {
Ll_str* next_ptr = ll_str_ptr->next;
free(ll_str_ptr);
ll_str_ptr = next_ptr;
}
chrom_info_ptr->incl_excl_name_stack = nullptr;
}
char* chrom_name_std(const Chrom_info* chrom_info_ptr, uint32_t chrom_idx, char* buf) {
const uint32_t output_encoding = chrom_info_ptr->output_encoding;
if (output_encoding & (CHR_OUTPUT_PREFIX | CHR_OUTPUT_0M)) {
if (output_encoding == CHR_OUTPUT_0M) {
// force two chars
if (chrom_idx <= chrom_info_ptr->autosome_ct) {
buf = (char*)memcpya(buf, &(digit2_table[chrom_idx * 2]), 2);
} else if ((int32_t)chrom_idx == chrom_info_ptr->xymt_codes[XY_OFFSET]) {
buf = (char*)memcpya(buf, "XY", 2);
} else {
*buf++ = '0';
if ((int32_t)chrom_idx == chrom_info_ptr->xymt_codes[X_OFFSET]) {
*buf++ = 'X';
} else {
// assumes only X/Y/XY/MT defined
*buf++ = ((int32_t)chrom_idx == chrom_info_ptr->xymt_codes[Y_OFFSET])? 'Y' : 'M';
}
}
return buf;
}
buf = memcpyl3a(buf, "chr");
}
if ((!(output_encoding & (CHR_OUTPUT_M | CHR_OUTPUT_MT))) || (chrom_idx <= chrom_info_ptr->autosome_ct)) {
return uint32toa(chrom_idx, buf);
} else if ((int32_t)chrom_idx == chrom_info_ptr->xymt_codes[X_OFFSET]) {
*buf++ = 'X';
} else if ((int32_t)chrom_idx == chrom_info_ptr->xymt_codes[Y_OFFSET]) {
*buf++ = 'Y';
} else if ((int32_t)chrom_idx == chrom_info_ptr->xymt_codes[XY_OFFSET]) {
buf = (char*)memcpya(buf, "XY", 2);
} else {
*buf++ = 'M';
if (output_encoding & CHR_OUTPUT_MT) {
*buf++ = 'T';
}
}
return buf;
}
char* chrom_name_write(const Chrom_info* chrom_info_ptr, uint32_t chrom_idx, char* buf) {
// assumes chrom_idx is valid
if (!chrom_idx) {
*buf++ = '0';
return buf;
} else if (chrom_idx <= chrom_info_ptr->max_code) {
return chrom_name_std(chrom_info_ptr, chrom_idx, buf);
} else if (chrom_info_ptr->zero_extra_chroms) {
*buf++ = '0';
return buf;
} else {
return strcpya(buf, chrom_info_ptr->nonstd_names[chrom_idx]);
}
}
char* chrom_name_buf5w4write(const Chrom_info* chrom_info_ptr, uint32_t chrom_idx, uint32_t* chrom_name_len_ptr, char* buf5) {
uint32_t slen;
*chrom_name_len_ptr = 4;
if (!chrom_idx) {
memcpy(buf5, " 0", 4);
} else if (chrom_idx <= chrom_info_ptr->max_code) {
if (chrom_info_ptr->output_encoding & CHR_OUTPUT_PREFIX) {
*chrom_name_len_ptr = (uintptr_t)(chrom_name_std(chrom_info_ptr, chrom_idx, buf5) - buf5);
} else {
width_force(4, buf5, chrom_name_std(chrom_info_ptr, chrom_idx, buf5));
}
} else if (chrom_info_ptr->zero_extra_chroms) {
memcpy(buf5, " 0", 4);
} else {
slen = strlen(chrom_info_ptr->nonstd_names[chrom_idx]);
if (slen < 4) {
fw_strcpyn(4, slen, chrom_info_ptr->nonstd_names[chrom_idx], buf5);
} else {
*chrom_name_len_ptr = slen;
return chrom_info_ptr->nonstd_names[chrom_idx];
}
}
return buf5;
}
uint32_t get_max_chrom_slen(const Chrom_info* chrom_info_ptr) {
// does not include trailing null
// can be overestimate
// if more functions start calling this, it should just be built into
// load_bim() instead
if (chrom_info_ptr->zero_extra_chroms) {
return 3 + MAX_CHROM_TEXTNUM_SLEN;
}
const uint32_t chrom_ct = chrom_info_ptr->chrom_ct;
const uint32_t max_code = chrom_info_ptr->max_code;
uint32_t max_chrom_slen = 3 + MAX_CHROM_TEXTNUM_SLEN;
for (uint32_t chrom_fo_idx = 0; chrom_fo_idx < chrom_ct; chrom_fo_idx++) {
const uint32_t chrom_idx = chrom_info_ptr->chrom_file_order[chrom_fo_idx];
if (!is_set(chrom_info_ptr->chrom_mask, chrom_idx)) {
continue;
}
if (chrom_idx > max_code) {
const uint32_t name_slen = strlen(chrom_info_ptr->nonstd_names[chrom_idx]);
if (name_slen > max_chrom_slen) {
max_chrom_slen = name_slen;
}
}
}
return max_chrom_slen;
}
uint32_t haploid_chrom_present(const Chrom_info* chrom_info_ptr) {
const uintptr_t* chrom_mask = chrom_info_ptr->chrom_mask;
const uintptr_t* haploid_mask = chrom_info_ptr->haploid_mask;
for (uint32_t widx = 0; widx < CHROM_MASK_INITIAL_WORDS; widx++) {
if (chrom_mask[widx] & haploid_mask[widx]) {
return 1;
}
}
return 0;
}
static inline int32_t single_letter_chrom(uint32_t letter) {
letter &= 0xdf;
if (letter == 'X') {
return CHROM_X;
} else if (letter == 'Y') {
return CHROM_Y;
} else if (letter == 'M') {
return CHROM_MT;
} else {
return -1;
}
}
int32_t get_chrom_code_raw(const char* sptr) {
// any character <= ' ' is considered a terminator
// note that char arithmetic tends to be compiled to int32 operations, so we
// mostly work with ints here
// assumes MAX_CHROM_TEXTNUM_SLEN == 2
uint32_t first_char_code = (unsigned char)sptr[0];
uint32_t second_char_code = (unsigned char)sptr[1];
if ((first_char_code & 0xdf) == 'C') {
if (((second_char_code & 0xdf) == 'H') && ((((unsigned char)sptr[2]) & 0xdf) == 'R')) {
sptr = &(sptr[3]);
first_char_code = (unsigned char)sptr[0];
second_char_code = (unsigned char)sptr[1];
} else {
return -1;
}
}
if (second_char_code > ' ') {
#if MAX_CHROM_TEXTNUM_SLEN == 2
if (sptr[2] > ' ') {
return -1;
}
#endif
const uint32_t first_char_toi = first_char_code - '0';
if (first_char_toi < 10) {
const uint32_t second_char_toi = second_char_code - '0';
if (second_char_toi < 10) {
uint32_t chrom_code = first_char_toi * 10 + second_char_toi;
#if MAX_CHROM_TEXTNUM_SLEN > 2
for (uint32_t pos = 2; pos < MAX_CHROM_TEXTNUM_SLEN; ++pos) {
const uint32_t char_code = (unsigned char)sptr[pos];
if (char_code <= ' ') {
return chrom_code;
}
const uint32_t char_toi = char_code - '0';
if (char_toi >= 10) {
return -1;
}
chrom_code = chrom_code * 10 + char_toi;
}
const uint32_t extra_char_code = (unsigned char)sptr[MAX_CHROM_TEXTNUM_SLEN];
if (extra_char_code > ' ') {
return -1;
}
#endif
return chrom_code;
} else if (!first_char_toi) {
// accept '0X', '0Y', '0M' emitted by Oxford software
return single_letter_chrom(second_char_code);
}
} else {
first_char_code &= 0xdf;
if (first_char_code == 'X') {
if ((second_char_code == 'Y') || (second_char_code == 'y')) {
return CHROM_XY;
}
} else if (first_char_code == 'M') {
if ((second_char_code == 'T') || (second_char_code == 't')) {
return CHROM_MT;
}
}
}
} else {
const uint32_t first_char_toi = first_char_code - '0';
if (first_char_toi < 10) {
return first_char_toi;
} else {
return single_letter_chrom(first_char_code);
}
}
return -1;
}
int32_t get_chrom_code(const char* chrom_name, const Chrom_info* chrom_info_ptr, uint32_t name_slen) {
// requires chrom_name to be null-terminated
// in practice, name_slen will usually already be known, may as well avoid
// redundant strlen() calls even though this uglifies the interface
// does not perform exhaustive error-checking
// -1 = --allow-extra-chr ok, -2 = total fail
const int32_t chrom_code_raw = get_chrom_code_raw(chrom_name);
if (((const uint32_t)chrom_code_raw) <= chrom_info_ptr->max_code) {
return chrom_code_raw;
}
if (chrom_code_raw != -1) {
if (chrom_code_raw >= MAX_POSSIBLE_CHROM) {
return chrom_info_ptr->xymt_codes[chrom_code_raw - MAX_POSSIBLE_CHROM];
}
return -2;
}
if (!chrom_info_ptr->name_ct) {
return -1;
}
// 0xffffffffU gets casted to -1
return (int32_t)nonstd_chrom_name_htable_find(chrom_name, (const char* const*)chrom_info_ptr->nonstd_names, chrom_info_ptr->nonstd_id_htable, name_slen);
}
int32_t get_chrom_code_counted(const Chrom_info* chrom_info_ptr, uint32_t name_slen, char* chrom_name) {
// when the chromosome name isn't null-terminated
char* s_end = &(chrom_name[name_slen]);
const char tmpc = *s_end;
*s_end = '\0';
const int32_t retval = get_chrom_code(chrom_name, chrom_info_ptr, name_slen);
*s_end = tmpc;
return retval;
}
uint32_t get_variant_chrom_fo_idx(const Chrom_info* chrom_info_ptr, uintptr_t variant_uidx) {
const uint32_t* variant_binsearch = chrom_info_ptr->chrom_fo_vidx_start;
uint32_t chrom_fo_min = 0;
uint32_t chrom_ct = chrom_info_ptr->chrom_ct;
while (chrom_ct - chrom_fo_min > 1) {
const uint32_t chrom_fo_cur = (chrom_ct + chrom_fo_min) / 2;
if (variant_binsearch[chrom_fo_cur] > variant_uidx) {
chrom_ct = chrom_fo_cur;
} else {
chrom_fo_min = chrom_fo_cur;
}
}
return chrom_fo_min;
}
void chrom_error(const char* chrom_name, const char* file_descrip, const Chrom_info* chrom_info_ptr, uintptr_t line_idx, int32_t error_code) {
// assumes chrom_name is null-terminated
const int32_t raw_code = get_chrom_code_raw(chrom_name);
logprint("\n");
if (line_idx) {
LOGERRPRINTFWW("Error: Invalid chromosome code '%s' on line %" PRIuPTR " of %s.\n", chrom_name, line_idx, file_descrip);
} else {
LOGERRPRINTFWW("Error: Invalid chromosome code '%s' in %s.\n", chrom_name, file_descrip);
}
if ((raw_code > ((int32_t)chrom_info_ptr->max_code)) && ((raw_code <= MAX_CHROM_TEXTNUM + XYMT_OFFSET_CT) || (raw_code >= MAX_POSSIBLE_CHROM))) {
if (chrom_info_ptr->species != SPECIES_UNKNOWN) {
if (chrom_info_ptr->species == SPECIES_HUMAN) {
logerrprint("(This is disallowed for humans. Check if the problem is with your data, or if\nyou forgot to define a different chromosome set with e.g. --chr-set.)\n");
} else {
logerrprint("(This is disallowed by the PLINK 1.07 species flag you used. You can\ntemporarily work around this restriction with --chr-set; contact the developers\nif you want the flag to be permanently redefined.)\n");
}
} else {
logerrprint("(This is disallowed by your --chr-set/--autosome-num parameters. Check if the\nproblem is with your data, or your command line.)\n");
}
} else if (error_code == -1) {
logerrprint("(Use --allow-extra-chr to force it to be accepted.)\n");
}
}
int32_t try_to_add_chrom_name(const char* chrom_name, const char* file_descrip, uintptr_t line_idx, uint32_t name_slen, uint32_t allow_extra_chroms, int32_t* chrom_idx_ptr, Chrom_info* chrom_info_ptr) {
// assumes chrom_name is nonstandard (i.e. not "2", "chr2", "chrX", etc.)
// requires chrom_name to be null-terminated
// assumes chrom_idx currently has the return value of get_chrom_code()
if ((!allow_extra_chroms) || ((*chrom_idx_ptr) == -2)) {
chrom_error(chrom_name, file_descrip, chrom_info_ptr, line_idx, *chrom_idx_ptr);
return RET_MALFORMED_INPUT;
}
// quasi-bugfix: remove redundant hash table check
if (chrom_name[0] == '#') {
// redundant with some of the comment-skipping loaders, but this isn't
// performance-critical
logprint("\n");
logerrprint("Error: Chromosome/contig names may not begin with '#'.\n");
return RET_MALFORMED_INPUT;
}
if (name_slen > MAX_ID_SLEN) {
logprint("\n");
if (line_idx) {
LOGERRPRINTFWW("Error: Line %" PRIuPTR " of %s has an excessively long chromosome/contig name. (The " PROG_NAME_CAPS " limit is " MAX_ID_SLEN_STR " characters.)\n", line_idx, file_descrip);
} else {
LOGERRPRINTFWW("Error: Excessively long chromosome/contig name in %s. (The " PROG_NAME_CAPS " limit is " MAX_ID_SLEN_STR " characters.)\n", file_descrip);
}
return RET_MALFORMED_INPUT;
}
const uint32_t max_code_p1 = chrom_info_ptr->max_code + 1;
const uint32_t name_ct = chrom_info_ptr->name_ct;
const uint32_t chrom_code_end = max_code_p1 + name_ct;
if (chrom_code_end == MAX_POSSIBLE_CHROM) {
logprint("\n");
logerrprint("Error: Too many distinct nonstandard chromosome/contig names.\n");
return RET_MALFORMED_INPUT;
}
if (!name_ct) {
// lazy initialization
fill_uint_one(CHROM_NAME_HTABLE_SIZE, chrom_info_ptr->nonstd_id_htable);
}
char** nonstd_names = chrom_info_ptr->nonstd_names;
nonstd_names[chrom_code_end] = (char*)malloc(name_slen + 1);
if (!nonstd_names[chrom_code_end]) {
return RET_NOMEM;
}
Ll_str* name_stack_ptr = chrom_info_ptr->incl_excl_name_stack;
uint32_t in_name_stack = 0;
while (name_stack_ptr) {
// there shouldn't be many of these, so sorting is unimportant
if (!strcmp(chrom_name, name_stack_ptr->ss)) {
in_name_stack = 1;
break;
}
name_stack_ptr = name_stack_ptr->next;
}
if ((in_name_stack && chrom_info_ptr->is_include_stack) || ((!in_name_stack) && (!chrom_info_ptr->is_include_stack))) {
SET_BIT(chrom_code_end, chrom_info_ptr->chrom_mask);
if (chrom_info_ptr->haploid_mask[0] & 1) {
SET_BIT(chrom_code_end, chrom_info_ptr->haploid_mask);
}
}
memcpy(nonstd_names[chrom_code_end], chrom_name, name_slen + 1);
*chrom_idx_ptr = (int32_t)chrom_code_end;
chrom_info_ptr->name_ct = name_ct + 1;
uint32_t* id_htable = chrom_info_ptr->nonstd_id_htable;
uint32_t hashval = murmurhash3_32(chrom_name, name_slen) % CHROM_NAME_HTABLE_SIZE;
uint32_t next_incr = 1;
while (1) {
if (id_htable[hashval] == 0xffffffffU) {
id_htable[hashval] = chrom_code_end;
return 0;
}
// no overflow danger here
hashval += next_incr;
if (hashval >= CHROM_NAME_HTABLE_SIZE) {
hashval -= CHROM_NAME_HTABLE_SIZE;
}
next_incr += 2; // quadratic probing
}
}
uint32_t allele_set(const char* newval, uint32_t slen, char** allele_ptr) {
char* newptr;
if (slen == 1) {
newptr = (char*)(&(g_one_char_strs[((unsigned char)*newval) * 2]));
} else {
newptr = (char*)malloc(slen + 1);
if (!newptr) {
return 1;
}
memcpyx(newptr, newval, slen, '\0');
}
*allele_ptr = newptr;
return 0;
}
uint32_t allele_reset(const char* newval, uint32_t slen, char** allele_ptr) {
char* newptr;
if (slen == 1) {
newptr = (char*)(&(g_one_char_strs[((uint8_t)*newval) * 2]));
} else {
newptr = (char*)malloc(slen + 1);
if (!newptr) {
return 1;
}
memcpyx(newptr, newval, slen, '\0');
}
if (allele_ptr[0][1]) {
free(*allele_ptr);
}
*allele_ptr = newptr;
return 0;
}
void cleanup_allele_storage(uint32_t max_allele_slen, uintptr_t allele_storage_entry_ct, char** allele_storage) {
if (allele_storage && (max_allele_slen > 1)) {
const uintptr_t one_char_strs_addr = (uintptr_t)g_one_char_strs;
for (uintptr_t idx = 0; idx < allele_storage_entry_ct; ++idx) {
char* cur_entry = allele_storage[idx];
assert(cur_entry);
// take advantage of unsigned wraparound
if ((((uintptr_t)cur_entry) - one_char_strs_addr) >= 512) {
free(cur_entry);
}
}
}
}
void cleanup_allele_storage2(uintptr_t allele_storage_entry_ct, char** allele_storage) {
if (allele_storage) {
const uintptr_t one_char_strs_addr = (uintptr_t)g_one_char_strs;
for (uintptr_t idx = 0; idx < allele_storage_entry_ct;) {
char* cur_entry = allele_storage[idx];
if (!cur_entry) {
// --merge-equal-pos hacked entry
idx += 2;
continue;
}
// take advantage of unsigned wraparound
if ((((uintptr_t)cur_entry) - one_char_strs_addr) >= 512) {
free(cur_entry);
}
++idx;
}
}
}
void refresh_chrom_info(const Chrom_info* chrom_info_ptr, uintptr_t marker_uidx, uint32_t* __restrict chrom_end_ptr, uint32_t* __restrict chrom_fo_idx_ptr, uint32_t* __restrict is_x_ptr, uint32_t* __restrict is_y_ptr, uint32_t* __restrict is_mt_ptr, uint32_t* __restrict is_haploid_ptr) {
// assumes we are at the end of the chromosome denoted by chrom_fo_idx. Ok
// for chrom_fo_idx == 0xffffffffU.
// assumes marker_uidx < unfiltered_marker_ct
*chrom_end_ptr = chrom_info_ptr->chrom_fo_vidx_start[(*chrom_fo_idx_ptr) + 1];
while (marker_uidx >= (*chrom_end_ptr)) {
*chrom_end_ptr = chrom_info_ptr->chrom_fo_vidx_start[(++(*chrom_fo_idx_ptr)) + 1];
}
const int32_t chrom_idx = chrom_info_ptr->chrom_file_order[*chrom_fo_idx_ptr];
*is_x_ptr = (chrom_idx == chrom_info_ptr->xymt_codes[X_OFFSET]);
*is_y_ptr = (chrom_idx == chrom_info_ptr->xymt_codes[Y_OFFSET]);
*is_mt_ptr = (chrom_idx == chrom_info_ptr->xymt_codes[MT_OFFSET]);
*is_haploid_ptr = is_set(chrom_info_ptr->haploid_mask, chrom_idx);
}
int32_t single_chrom_start(const Chrom_info* chrom_info_ptr, const uintptr_t* marker_exclude, uint32_t unfiltered_marker_ct) {
// Assumes there is at least one marker, and there are no split chromosomes.
// Returns first marker_uidx in chromosome if there is only one, or -1 if
// there's more than one chromosome.
uint32_t first_marker_uidx = next_unset_unsafe(marker_exclude, 0);
uint32_t last_marker_chrom = get_variant_chrom(chrom_info_ptr, last_clear_bit(marker_exclude, unfiltered_marker_ct));
return (get_variant_chrom(chrom_info_ptr, first_marker_uidx) == last_marker_chrom)? first_marker_uidx : -1;
}
double get_dmedian(const double* sorted_arr, uintptr_t len) {
if (len) {
if (len % 2) {
return sorted_arr[len / 2];
} else {
return (sorted_arr[len / 2] + sorted_arr[(len / 2) - 1]) * 0.5;
}
} else {
return 0.0;
}
}
#ifdef __cplusplus
double destructive_get_dmedian(uintptr_t len, double* unsorted_arr) {
if (!len) {
return 0.0;
}
uintptr_t len_d2 = len / 2;
std::nth_element(unsorted_arr, &(unsorted_arr[len_d2]), &(unsorted_arr[len]));
if (!(len % 2)) {
std::nth_element(unsorted_arr, &(unsorted_arr[len_d2 - 1]), &(unsorted_arr[len_d2]));
return (unsorted_arr[len_d2 - 1] + unsorted_arr[len_d2]) * 0.5;
} else {
return unsorted_arr[len_d2];
}
}
#else
double destructive_get_dmedian(uintptr_t len, double* unsorted_arr) {
// no, I'm not gonna bother reimplementing introselect just for folks who
// insist on using gcc over g++
qsort(unsorted_arr, len, sizeof(double), double_cmp);
return get_dmedian(unsorted_arr, len);
}
#endif
int32_t strcmp_casted(const void* s1, const void* s2) {
return strcmp((char*)s1, (char*)s2);
}
// PLINK 2's natural sort uses the following logic:
// - All alphabetic characters act as if they are capitalized, except for
// tiebreaking purposes (where ASCII is used).
// - Numbers are compared by magnitude, with the exception of...
// - Numbers with leading zero(es). If you're putting extraneous zeroes in
// front of IDs, we assume they're there to force particular items to be sorted
// earlier, rather than just appearing at random. So, unlike many natural sort
// implementations, we sort 00200 < 021 < 20: all numbers with n leading zeroes
// are sorted before all numbers with (n-1) leading zeroes; magnitude only
// applies if the leading zero counts match. This handles e.g. subbasement
// room numbering properly.
//
// This won't always do what you want if your IDs have variable-length decimals
// in them (e.g. it yields 0.99 < 0.101); if you don't want to fall back on
// ASCII sort, enforce a fixed number of digits after the decimal point. Also
// note that ASCII sort is outright better for e.g. numbers represented in
// hexadecimal or base 36. In principle, it's possible to reliably autodetect
// some of these cases (especially hexadecimal numbers beginning with "0x"),
// but that'll never be perfect so we just let the user toggle the sort method.
int32_t strcmp_natural_scan_forward(const unsigned char* s1, const unsigned char* s2) {
// assumes s1 and s2 currently point to the middle of a mismatching number,
// where s1 < s2.
unsigned char c1;
unsigned char c2;
do {
c1 = *(++s1);
c2 = *(++s2);
if (is_not_digit(c1)) {
return -1;
}
} while (is_digit(c2));
return 1;
}
// We have the following major states:
// 0 (initial): strings perfectly match so far, last char (if any) is
// nonnumeric.
// 1: strings perfectly match so far, last char is numeric.
// 2: strings match except for capitalization, last char is nonnumeric.
// 3: strings match except for capitalization, last char is numeric.
// strcmp_natural_tiebroken() expresses the logic for states 2 and 3, while
// strcmp_natural_uncasted() handles states 0 and 1.
int32_t strcmp_natural_tiebroken(const unsigned char* s1, const unsigned char* s2) {
// assumes ties should be broken in favor of s2.
unsigned char c1 = *(++s1);
unsigned char c2 = *(++s2);
while (is_not_nzdigit(c1) && is_not_nzdigit(c2)) {
// state 2
strcmp_natural_tiebroken_state_2:
if (c1 != c2) {
if ((c1 >= 'a') && (c1 <= 'z')) {
c1 -= 32;
}
if ((c2 >= 'a') && (c2 <= 'z')) {
c2 -= 32;
}
if (c1 < c2) {
return -1;
} else if (c1 > c2) {
return 1;
}
} else if (!c1) {
return -1;
}
c1 = *(++s1);
c2 = *(++s2);
}
if (is_not_nzdigit(c1) || is_not_nzdigit(c2)) {
return (c1 < c2)? -1 : 1;
}
do {
// state 3
if (c1 != c2) {
if (is_digit(c2)) {
if (c1 < c2) {
return strcmp_natural_scan_forward(s1, s2);
} else {
return -strcmp_natural_scan_forward(s2, s1);
}
}
return 1;
}
c1 = *(++s1);
c2 = *(++s2);
} while (is_digit(c1));
if (is_digit(c2)) {
return -1;
}
// skip the while (is_not_digit...) check
goto strcmp_natural_tiebroken_state_2;
}
static inline int32_t strcmp_natural_uncasted(const unsigned char* s1, const unsigned char* s2) {
unsigned char c1 = *s1;
unsigned char c2 = *s2;
while (is_not_nzdigit(c1) && is_not_nzdigit(c2)) {
// state 0
strcmp_natural_uncasted_state_0:
if (c1 != c2) {
if ((c1 >= 'a') && (c1 <= 'z')) {
if (c2 + 32 == c1) {
return -strcmp_natural_tiebroken(s2, s1);
} else if ((c2 < 'a') || (c2 > 'z')) {
c1 -= 32;
}
} else if ((c2 >= 'a') && (c2 <= 'z')) {
c2 -= 32;
if (c1 == c2) {
return strcmp_natural_tiebroken(s1, s2);
}
}
return (c1 < c2)? -1 : 1;
} else if (!c1) {
return 0;
}
c1 = *(++s1);
c2 = *(++s2);
}
if (is_not_nzdigit(c1) || is_not_nzdigit(c2)) {
return (c1 < c2)? -1 : 1;
}
do {
// state 1
if (c1 != c2) {
if (is_digit(c2)) {
if (c1 < c2) {
return strcmp_natural_scan_forward(s1, s2);
} else {
return -strcmp_natural_scan_forward(s2, s1);
}
}
return 1;
}
c1 = *(++s1);
c2 = *(++s2);
} while (is_digit(c1));
if (is_digit(c2)) {
return -1;
}
goto strcmp_natural_uncasted_state_0;
}
int32_t strcmp_natural(const void* s1, const void* s2) {
return strcmp_natural_uncasted((unsigned char*)s1, (unsigned char*)s2);
}
int32_t strcmp_deref(const void* s1, const void* s2) {
return strcmp(*(char**)s1, *(char**)s2);
}
int32_t strcmp_natural_deref(const void* s1, const void* s2) {
return strcmp_natural_uncasted(*(unsigned char**)s1, *(unsigned char**)s2);
}
int32_t get_uidx_from_unsorted(const char* idstr, const uintptr_t* exclude_arr, uint32_t id_ct, const char* unsorted_ids, uintptr_t max_id_len) {
uintptr_t id_uidx = 0;
uintptr_t slen_p1 = strlen(idstr) + 1;
uint32_t id_idx;
if (slen_p1 > max_id_len) {
return -1;
}
for (id_idx = 0; id_idx < id_ct; id_uidx++, id_idx++) {
id_uidx = next_unset_ul_unsafe(exclude_arr, id_uidx);
if (!memcmp(idstr, &(unsorted_ids[id_uidx * max_id_len]), slen_p1)) {
return (int32_t)((uint32_t)id_uidx);
}
}
return -1;
}
char* scan_for_duplicate_ids(char* sorted_ids, uintptr_t id_ct, uintptr_t max_id_len) {
uintptr_t id_idx;
id_ct--;
for (id_idx = 0; id_idx < id_ct; id_idx++) {
if (!strcmp(&(sorted_ids[id_idx * max_id_len]), &(sorted_ids[(id_idx + 1) * max_id_len]))) {
return &(sorted_ids[id_idx * max_id_len]);
}
}
return nullptr;
}
char* scan_for_duplicate_or_overlap_ids(char* sorted_ids, uintptr_t id_ct, uintptr_t max_id_len, const char* sorted_nonoverlap_ids, uintptr_t nonoverlap_id_ct, uintptr_t max_nonoverlap_id_len) {
// extended scan_for_duplicate_ids() which also verifies that no entry in
// sorted_ids matches any entry in sorted_nonoverlap_ids.
// nonoverlap_id_ct == 0 and sorted_nonoverlap_ids == nullptr ok. id_ct
// cannot be zero, though.
uintptr_t nonoverlap_id_idx = 0;
uintptr_t id_idx = 0;
char* cur_id_ptr = sorted_ids;
const char* nonoverlap_id_ptr;
char* other_id_ptr;
int32_t ii;
while (1) {
if (nonoverlap_id_idx == nonoverlap_id_ct) {
return scan_for_duplicate_ids(cur_id_ptr, id_ct - id_idx, max_id_len);
}
nonoverlap_id_ptr = &(sorted_nonoverlap_ids[nonoverlap_id_idx * max_nonoverlap_id_len]);
ii = strcmp(cur_id_ptr, nonoverlap_id_ptr);
if (ii < 0) {
if (++id_idx == id_ct) {
return nullptr;
}
other_id_ptr = &(cur_id_ptr[max_id_len]);
if (!strcmp(cur_id_ptr, other_id_ptr)) {
return cur_id_ptr;
}
cur_id_ptr = other_id_ptr;
continue;
} else if (!ii) {
return cur_id_ptr;
}
nonoverlap_id_idx++;
}
}
int32_t eval_affection(const char* bufptr, double missing_phenod) {
// turns out --1 had the side-effect of *forcing* case/control
// interpretation in 1.07. We replicate that for backward compatibility, and
// no longer call this function in that context.
char* ss;
double dxx;
// this used to be an integer read, but that could do the wrong thing if e.g.
// all phenotypes were -9.xxx...
dxx = strtod(bufptr, &ss);
if ((ss == bufptr) || (dxx == missing_phenod)) {
return 1;
}
return ((bufptr[0] == '0') || (bufptr[0] == '1') || (bufptr[0] == '2')) && is_space_or_eoln(bufptr[1]);
}
uint32_t triangle_divide(int64_t cur_prod, int32_t modif) {
// return smallest integer vv for which (vv * (vv + modif)) is no smaller
// than cur_prod, and neither term in the product is negative. (Note the
// lack of a divide by two; cur_prod should also be double its "true" value
// as a result.)
int64_t vv;
if (cur_prod == 0) {
if (modif < 0) {
return -modif;
} else {
return 0;
}
}
vv = (int64_t)sqrt((double)cur_prod);
while ((vv - 1) * (vv + modif - 1) >= cur_prod) {
vv--;
}
while (vv * (vv + modif) < cur_prod) {
vv++;
}
return vv;
}
void parallel_bounds(uint32_t ct, int32_t start, uint32_t parallel_idx, uint32_t parallel_tot, int32_t* __restrict bound_start_ptr, int32_t* __restrict bound_end_ptr) {
int32_t modif = 1 - start * 2;
int64_t ct_tot = ((int64_t)ct) * (ct + modif);
*bound_start_ptr = triangle_divide((ct_tot * parallel_idx) / parallel_tot, modif);
*bound_end_ptr = triangle_divide((ct_tot * (parallel_idx + 1)) / parallel_tot, modif);
}
// this might belong in plink_calc instead, not being used anywhere else
// set align to 1 for no alignment
void triangle_fill(uint32_t ct, uint32_t pieces, uint32_t parallel_idx, uint32_t parallel_tot, uint32_t start, uint32_t align, uint32_t* target_arr) {
int32_t modif = 1 - start * 2;
uint32_t cur_piece = 1;
int64_t ct_tr;
int64_t cur_prod;
int32_t lbound;
int32_t ubound;
uint32_t uii;
uint32_t align_m1;
parallel_bounds(ct, start, parallel_idx, parallel_tot, &lbound, &ubound);
// x(x+1)/2 is divisible by y iff (x % (2y)) is 0 or (2y - 1).
align *= 2;
align_m1 = align - 1;
target_arr[0] = lbound;
target_arr[pieces] = ubound;
cur_prod = ((int64_t)lbound) * (lbound + modif);
ct_tr = (((int64_t)ubound) * (ubound + modif) - cur_prod) / pieces;
while (cur_piece < pieces) {
cur_prod += ct_tr;
lbound = triangle_divide(cur_prod, modif);
uii = (lbound - ((int32_t)start)) & align_m1;
if ((uii) && (uii != align_m1)) {
lbound = start + ((lbound - ((int32_t)start)) | align_m1);
}
// lack of this check caused a nasty bug earlier
if (((uint32_t)lbound) > ct) {
lbound = ct;
}
target_arr[cur_piece++] = lbound;
}
}
int32_t relationship_req(uint64_t calculation_type) {
return (calculation_type & (CALC_RELATIONSHIP | CALC_UNRELATED_HERITABILITY | CALC_REL_CUTOFF | CALC_REGRESS_REL | CALC_PCA))? 1 : 0;
}
int32_t distance_req(const char* read_dists_fname, uint64_t calculation_type) {
return ((calculation_type & CALC_DISTANCE) || ((calculation_type & (CALC_PLINK1_DISTANCE_MATRIX | CALC_PLINK1_IBS_MATRIX)) && (!(calculation_type & CALC_GENOME))) || ((!read_dists_fname) && (calculation_type & (CALC_IBS_TEST | CALC_GROUPDIST | CALC_REGRESS_DISTANCE))));
}
int32_t double_cmp(const void* aa, const void* bb) {
double cc = *((const double*)aa) - *((const double*)bb);
if (cc > 0.0) {
return 1;
} else if (cc < 0.0) {
return -1;
} else {
return 0;
}
}
int32_t double_cmp_decr(const void* aa, const void* bb) {
double cc = *((const double*)aa) - *((const double*)bb);
if (cc < 0.0) {
return 1;
} else if (cc > 0.0) {
return -1;
} else {
return 0;
}
}
int32_t double_cmp_deref(const void* aa, const void* bb) {
double cc = **((const double**)aa) - **((const double**)bb);
if (cc > 0.0) {
return 1;
} else if (cc < 0.0) {
return -1;
} else {
return 0;
}
}
int32_t char_cmp_deref(const void* aa, const void* bb) {
return (int32_t)(**((const char**)aa) - **((const char**)bb));
}
int32_t double_cmp_deref_tiebreak(const void* aa, const void* bb) {
double cc = **((const double**)aa) - **((const double**)bb);
if (cc > 0.0) {
return 1;
} else if (cc < 0.0) {
return -1;
}
return ((const int32_t*)aa)[BYTECT4] - ((const int32_t*)bb)[BYTECT4];
}
int32_t intcmp(const void* aa, const void* bb) {
return *((const int32_t*)aa) - *((const int32_t*)bb);
}
int32_t uintcmp(const void* aa, const void* bb) {
if (*((const uint32_t*)aa) < *((const uint32_t*)bb)) {
return -1;
} else {
return (*((const uint32_t*)aa) > *((const uint32_t*)bb));
}
}
int32_t intcmp2(const void* aa, const void* bb) {
if (*((const int32_t*)aa) < *((const int32_t*)bb)) {
return -1;
} else {
return (*((const int32_t*)aa) > *((const int32_t*)bb));
}
}
int32_t intcmp3_decr(const void* aa, const void* bb) {
int32_t ii = *((const int32_t*)bb) - *((const int32_t*)aa);
if (ii) {
return ii;
}
ii = ((const int32_t*)bb)[1] - ((const int32_t*)aa)[1];
if (ii) {
return ii;
}
return ((const int32_t*)bb)[2] - ((const int32_t*)aa)[2];
}
#ifndef __cplusplus
int32_t llcmp(const void* aa, const void* bb) {
int64_t diff = *((const int64_t*)aa) - *((const int64_t*)bb);
if (diff > 0) {
return 1;
} else if (diff < 0) {
return -1;
} else {
return 0;
}
}
#endif
// alas, qsort_r not available on some Linux distributions
// Normally use qsort_ext(), but this version is necessary before g_bigstack
// has been allocated.
void qsort_ext2(char* main_arr, uintptr_t arr_length, uintptr_t item_length, int(* comparator_deref)(const void*, const void*), char* secondary_arr, uintptr_t secondary_item_len, char* proxy_arr, uintptr_t proxy_len) {
uintptr_t ulii;
for (ulii = 0; ulii < arr_length; ulii++) {
*(char**)(&(proxy_arr[ulii * proxy_len])) = &(main_arr[ulii * item_length]);
memcpy(&(proxy_arr[ulii * proxy_len + sizeof(void*)]), &(secondary_arr[ulii * secondary_item_len]), secondary_item_len);
}
qsort(proxy_arr, arr_length, proxy_len, comparator_deref);
for (ulii = 0; ulii < arr_length; ulii++) {
memcpy(&(secondary_arr[ulii * secondary_item_len]), &(proxy_arr[ulii * proxy_len + sizeof(void*)]), secondary_item_len);
memcpy(&(proxy_arr[ulii * proxy_len]), *(char**)(&(proxy_arr[ulii * proxy_len])), item_length);
}
for (ulii = 0; ulii < arr_length; ulii++) {
memcpy(&(main_arr[ulii * item_length]), &(proxy_arr[ulii * proxy_len]), item_length);
}
}
// This actually tends to be faster than just sorting an array of indices,
// because of memory locality issues.
int32_t qsort_ext(char* main_arr, uintptr_t arr_length, uintptr_t item_length, int(* comparator_deref)(const void*, const void*), char* secondary_arr, intptr_t secondary_item_len) {
// main_arr = packed array of equal-length items to sort
// arr_length = number of items
// item_length = byte count of each main_arr item
// comparator_deref = returns positive if *first > *second, 0 if equal,
// negative if *first < *second. Note the extra
// dereference.
// secondary_arr = packed array of fixed-length records associated with the
// main_arr items, to be resorted in the same way. (e.g.
// if one is building an index, this could start as a sorted
// 0..(n-1) sequence of integers; then, post-sort, this would
// be a lookup table for the original position of each
// main_arr item.)
// secondary_item_len = byte count of each secondary_arr item
uintptr_t proxy_len = secondary_item_len + sizeof(void*);
unsigned char* bigstack_mark = g_bigstack_base;
char* proxy_arr;
if (!arr_length) {
return 0;
}
if (proxy_len < item_length) {
proxy_len = item_length;
}
if (bigstack_alloc_c(arr_length * proxy_len, &proxy_arr)) {
return -1;
}
qsort_ext2(main_arr, arr_length, item_length, comparator_deref, secondary_arr, secondary_item_len, proxy_arr, proxy_len);
bigstack_reset(bigstack_mark);
return 0;
}
int32_t sort_item_ids_noalloc(uintptr_t unfiltered_ct, const uintptr_t* exclude_arr, uintptr_t item_ct, const char* __restrict item_ids, uintptr_t max_id_len, uint32_t allow_dups, uint32_t collapse_idxs, int(* comparator_deref)(const void*, const void*), char* __restrict sorted_ids, uint32_t* id_map) {
// Stores a lexicographically sorted list of IDs in sorted_ids and the raw
// positions of the corresponding markers/samples in *id_map_ptr. Does not
// include excluded markers/samples in the list.
// Assumes sorted_ids and id_map have been allocated; use the sort_item_ids()
// wrapper if they haven't been.
// Note that this DOES still perform a "stack" allocation (in the qsort_ext()
// call).
uint32_t uii = 0;
char* dup_id;
char* tptr;
uint32_t ujj;
if (!item_ct) {
return 0;
}
if (!collapse_idxs) {
for (ujj = 0; ujj < item_ct; uii++, ujj++) {
next_unset_unsafe_ck(exclude_arr, &uii);
memcpy(&(sorted_ids[ujj * max_id_len]), &(item_ids[uii * max_id_len]), max_id_len);
id_map[ujj] = uii;
}
} else {
for (ujj = 0; ujj < item_ct; uii++, ujj++) {
next_unset_unsafe_ck(exclude_arr, &uii);
memcpy(&(sorted_ids[ujj * max_id_len]), &(item_ids[uii * max_id_len]), max_id_len);
id_map[ujj] = ujj;
}
}
if (qsort_ext(sorted_ids, item_ct, max_id_len, comparator_deref, (char*)id_map, sizeof(int32_t))) {
return RET_NOMEM;
}
if (!allow_dups) {
dup_id = scan_for_duplicate_ids(sorted_ids, item_ct, max_id_len);
if (dup_id) {
tptr = strchr(dup_id, '\t');
if (tptr) {
*tptr = ' ';
}
LOGERRPRINTFWW("Error: Duplicate ID '%s'.\n", dup_id);
return RET_INVALID_FORMAT;
}
}
return 0;
}
int32_t sort_item_ids(uintptr_t unfiltered_ct, const uintptr_t* exclude_arr, uintptr_t exclude_ct, const char* __restrict item_ids, uintptr_t max_id_len, uint32_t allow_dups, uint32_t collapse_idxs, int(* comparator_deref)(const void*, const void*), char** sorted_ids_ptr, uint32_t** id_map_ptr) {
uintptr_t item_ct = unfiltered_ct - exclude_ct;
// id_map on bottom because --indiv-sort frees *sorted_ids_ptr
if (bigstack_alloc_ui(item_ct, id_map_ptr) ||
bigstack_alloc_c(item_ct * max_id_len, sorted_ids_ptr)) {
return RET_NOMEM;
}
return sort_item_ids_noalloc(unfiltered_ct, exclude_arr, item_ct, item_ids, max_id_len, allow_dups, collapse_idxs, comparator_deref, *sorted_ids_ptr, *id_map_ptr);
}
uint32_t uint32arr_greater_than(const uint32_t* sorted_uint32_arr, uint32_t arr_length, uint32_t uii) {
// assumes arr_length is nonzero, and sorted_uint32_arr is in nondecreasing
// order. (useful for searching marker_pos.)
// uii guaranteed to be larger than sorted_uint32_arr[min_idx - 1] if it
// exists, but NOT necessarily sorted_uint32_arr[min_idx].
int32_t min_idx = 0;
// similarly, uii guaranteed to be no greater than
// sorted_uint32_arr[max_idx + 1] if it exists, but not necessarily
// sorted_uint32_arr[max_idx]. Signed integer since it could become -1.
int32_t max_idx = arr_length - 1;
uint32_t mid_idx;
while (min_idx < max_idx) {
mid_idx = (((uint32_t)min_idx) + ((uint32_t)max_idx)) / 2;
if (uii > sorted_uint32_arr[mid_idx]) {
min_idx = mid_idx + 1;
} else {
max_idx = mid_idx - 1;
}
}
if (uii > sorted_uint32_arr[((uint32_t)min_idx)]) {
return (min_idx + 1);
} else {
return min_idx;
}
}
uint32_t int32arr_greater_than(const int32_t* sorted_int32_arr, uint32_t arr_length, int32_t ii) {
int32_t min_idx = 0;
int32_t max_idx = arr_length - 1;
uint32_t mid_idx;
while (min_idx < max_idx) {
mid_idx = (((uint32_t)min_idx) + ((uint32_t)max_idx)) / 2;
if (ii > sorted_int32_arr[mid_idx]) {
min_idx = mid_idx + 1;
} else {
max_idx = mid_idx - 1;
}
}
if (ii > sorted_int32_arr[((uint32_t)min_idx)]) {
return (min_idx + 1);
} else {
return min_idx;
}
}
uintptr_t uint64arr_greater_than(const uint64_t* sorted_uint64_arr, uintptr_t arr_length, uint64_t ullii) {
intptr_t min_idx = 0;
intptr_t max_idx = arr_length - 1;
uintptr_t mid_idx;
while (min_idx < max_idx) {
mid_idx = (((uintptr_t)min_idx) + ((uintptr_t)max_idx)) / 2;
if (ullii > sorted_uint64_arr[mid_idx]) {
min_idx = mid_idx + 1;
} else {
max_idx = mid_idx - 1;
}
}
if (ullii > sorted_uint64_arr[((uintptr_t)min_idx)]) {
return (min_idx + 1);
} else {
return min_idx;
}
}
uintptr_t doublearr_greater_than(const double* sorted_dbl_arr, uintptr_t arr_length, double dxx) {
// returns number of items in sorted_dbl_arr which dxx is greater than.
// assumes array is nonempty and sorted in nondecreasing order
intptr_t min_idx = 0;
intptr_t max_idx = arr_length - 1;
uintptr_t mid_idx;
while (min_idx < max_idx) {
mid_idx = (((uintptr_t)min_idx) + ((uintptr_t)max_idx)) / 2;
if (dxx > sorted_dbl_arr[mid_idx]) {
min_idx = mid_idx + 1;
} else {
max_idx = mid_idx - 1;
}
}
if (dxx > sorted_dbl_arr[((uintptr_t)min_idx)]) {
return (min_idx + 1);
} else {
return min_idx;
}
}
uintptr_t nonincr_doublearr_leq_stride(const double* nonincr_dbl_arr, uintptr_t arr_length, uintptr_t stride, double dxx) {
// assumes relevant elements of array are sorted in nonincreasing order
// instead, and they are spaced stride units apart
intptr_t min_idx = 0;
intptr_t max_idx = arr_length - 1;
uintptr_t mid_idx;
while (min_idx < max_idx) {
mid_idx = (((uintptr_t)min_idx) + ((uintptr_t)max_idx)) / 2;
if (dxx <= nonincr_dbl_arr[mid_idx * stride]) {
min_idx = mid_idx + 1;
} else {
max_idx = mid_idx - 1;
}
}
if (dxx <= nonincr_dbl_arr[((uintptr_t)min_idx) * stride]) {
return (min_idx + 1);
} else {
return min_idx;
}
}
int32_t bsearch_str(const char* id_buf, uintptr_t cur_id_len, const char* lptr, uintptr_t max_id_len, uintptr_t end_idx) {
// does not assume null-terminated id_buf, or nonempty array.
// N.B. max_id_len includes null terminator as usual, while cur_id_len does
// NOT.
uintptr_t start_idx = 0;
uintptr_t mid_idx;
int32_t ii;
if (cur_id_len >= max_id_len) {
return -1;
}
while (start_idx < end_idx) {
mid_idx = (start_idx + end_idx) / 2;
ii = memcmp(id_buf, &(lptr[mid_idx * max_id_len]), cur_id_len);
if (ii > 0) {
start_idx = mid_idx + 1;
} else if ((ii < 0) || lptr[mid_idx * max_id_len + cur_id_len]) {
end_idx = mid_idx;
} else {
return ((uint32_t)mid_idx);
}
}
return -1;
}
int32_t bsearch_str_natural(const char* id_buf, const char* lptr, uintptr_t max_id_len, uintptr_t end_idx) {
// unlike bsearch_str(), caller is responsible for slen > max_id_len check
// if appropriate here
uintptr_t start_idx = 0;
uintptr_t mid_idx;
int32_t ii;
while (start_idx < end_idx) {
mid_idx = (start_idx + end_idx) / 2;
ii = strcmp_natural(id_buf, &(lptr[mid_idx * max_id_len]));
if (ii > 0) {
start_idx = mid_idx + 1;
} else if (ii < 0) {
end_idx = mid_idx;
} else {
return ((uint32_t)mid_idx);
}
}
return -1;
}
uintptr_t bsearch_str_lb(const char* id_buf, uintptr_t cur_id_len, const char* lptr, uintptr_t max_id_len, uintptr_t end_idx) {
// returns number of elements in lptr[] less than id_buf.
uintptr_t start_idx = 0;
uintptr_t mid_idx;
if (cur_id_len > max_id_len) {
cur_id_len = max_id_len;
}
while (start_idx < end_idx) {
mid_idx = (start_idx + end_idx) / 2;
if (memcmp(id_buf, &(lptr[mid_idx * max_id_len]), cur_id_len) > 0) {
start_idx = mid_idx + 1;
} else {
end_idx = mid_idx;
}
}
return start_idx;
}
uint32_t bsearch_read_fam_indiv(char* __restrict read_ptr, const char* __restrict lptr, uintptr_t max_id_len, uintptr_t filter_line_ct, char** read_pp_new, int32_t* retval_ptr, char* __restrict id_buf) {
// id_buf = workspace
// lptr = packed, sorted list of ID strings to search over
// read_ptr is assumed to point to beginning of FID. FID is terminated by
// any space/eoln character, then IID is assumed to follow it (and is also
// terminated by any space/eoln). Nonzero error value is returned if IID
// does not exist.
char* iid_ptr;
uintptr_t slen_fid;
uintptr_t slen_iid;
uintptr_t slen_final;
slen_fid = strlen_se(read_ptr);
iid_ptr = skip_initial_spaces(&(read_ptr[slen_fid]));
if (is_eoln_kns(*iid_ptr)) {
return 1;
}
slen_iid = strlen_se(iid_ptr);
if (read_pp_new) {
*read_pp_new = skip_initial_spaces(&(iid_ptr[slen_iid]));
}
slen_final = slen_fid + slen_iid + 1;
if (slen_final >= max_id_len) {
// avoid buffer overflow
*retval_ptr = -1;
return 0;
}
// error message bugfix: null-terminate this string
memcpyx(memcpyax(id_buf, read_ptr, slen_fid, '\t'), iid_ptr, slen_iid, '\0');
*retval_ptr = bsearch_str(id_buf, slen_final, lptr, max_id_len, filter_line_ct);
return 0;
}
void bsearch_fam(const char* __restrict fam_id, const char* __restrict lptr, uintptr_t max_id_len, uint32_t filter_line_ct, uint32_t* __restrict first_idx_ptr, uint32_t* __restrict last_idx_ptr, char* __restrict id_buf) {
uint32_t slen;
uint32_t fidx;
uint32_t loff;
if (!filter_line_ct) {
goto bsearch_fam_ret_null;
}
slen = strlen_se(fam_id);
if (slen + 3 > max_id_len) {
goto bsearch_fam_ret_null;
}
memcpy(id_buf, fam_id, slen);
id_buf[slen] = '\t';
fidx = bsearch_str_lb(id_buf, slen + 1, lptr, max_id_len, filter_line_ct);
if (fidx == filter_line_ct) {
goto bsearch_fam_ret_null;
}
id_buf[slen] = ' ';
loff = bsearch_str_lb(id_buf, slen + 1, &(lptr[fidx * max_id_len]), max_id_len, filter_line_ct - fidx);
if (!loff) {
goto bsearch_fam_ret_null;
}
*first_idx_ptr = fidx;
*last_idx_ptr = fidx + loff;
return;
bsearch_fam_ret_null:
*first_idx_ptr = 0;
*last_idx_ptr = 0;
}
void bitarr_invert(uintptr_t bit_ct, uintptr_t* bitarr) {
uintptr_t* bitarr_stop = &(bitarr[bit_ct / BITCT]);
while (bitarr < bitarr_stop) {
*bitarr = ~(*bitarr);
bitarr++;
}
if (bit_ct % BITCT) {
*bitarr = (~(*bitarr)) & ((ONELU << (bit_ct % BITCT)) - ONELU);
}
}
void bitarr_invert_copy(const uintptr_t* input_bitarr, uintptr_t bit_ct, uintptr_t* output_bitarr) {
const uintptr_t* input_stop = &(input_bitarr[bit_ct / BITCT]);
while (input_bitarr < input_stop) {
*output_bitarr++ = ~(*input_bitarr++);
}
if (bit_ct % BITCT) {
*output_bitarr = (~(*input_bitarr)) & ((ONELU << (bit_ct % BITCT)) - ONELU);
}
}
void bitvec_and(const uintptr_t* __restrict arg_bitvec, uintptr_t word_ct, uintptr_t* __restrict main_bitvec) {
// main_bitvec := main_bitvec AND arg_bitvec
#ifdef __LP64__
__m128i* vv128 = (__m128i*)main_bitvec;
const __m128i* iv128 = (const __m128i*)arg_bitvec;
__m128i* vv128_end = &(vv128[word_ct / 2]);
while (vv128 < vv128_end) {
*vv128 = _mm_and_si128(*iv128++, *vv128);
vv128++;
}
if (word_ct & 1) {
word_ct--;
main_bitvec[word_ct] &= arg_bitvec[word_ct];
}
#else
uintptr_t* bitvec_end = &(main_bitvec[word_ct]);
while (main_bitvec < bitvec_end) {
*main_bitvec++ &= *arg_bitvec++;
}
#endif
}
void bitvec_andnot(const uintptr_t* __restrict exclude_bitvec, uintptr_t word_ct, uintptr_t* __restrict main_bitvec) {
// main_bitvec := main_bitvec ANDNOT exclude_bitvec
// note that this is the reverse of the _mm_andnot() operand order
#ifdef __LP64__
__m128i* vv128 = (__m128i*)main_bitvec;
const __m128i* ev128 = (const __m128i*)exclude_bitvec;
__m128i* vv128_end = &(vv128[word_ct / 2]);
while (vv128 < vv128_end) {
*vv128 = _mm_andnot_si128(*ev128++, *vv128);
vv128++;
}
if (word_ct & 1) {
word_ct--;
main_bitvec[word_ct] &= ~(exclude_bitvec[word_ct]);
}
#else
uintptr_t* bitvec_end = &(main_bitvec[word_ct]);
while (main_bitvec < bitvec_end) {
*main_bitvec++ &= ~(*exclude_bitvec++);
}
#endif
}
void bitvec_andnot_reversed_args(const uintptr_t* __restrict include_bitvec, uintptr_t word_ct, uintptr_t* __restrict main_bitvec) {
// main_bitvec := (~main_bitvec) AND include_bitvec
#ifdef __LP64__
__m128i* vv128 = (__m128i*)main_bitvec;
const __m128i* iv128 = (const __m128i*)include_bitvec;
__m128i* vv128_end = &(vv128[word_ct / 2]);
while (vv128 < vv128_end) {
*vv128 = _mm_andnot_si128(*vv128, *iv128++);
vv128++;
}
if (word_ct & 1) {
word_ct--;
main_bitvec[word_ct] = (~main_bitvec[word_ct]) & include_bitvec[word_ct];
}
#else
uintptr_t* bitvec_end = &(main_bitvec[word_ct]);
while (main_bitvec < bitvec_end) {
*main_bitvec = (~(*main_bitvec)) & (*include_bitvec++);
main_bitvec++;
}
#endif
}
void bitvec_or(const uintptr_t* __restrict arg_bitvec, uintptr_t word_ct, uintptr_t* main_bitvec) {
// main_bitvec := main_bitvec OR arg_bitvec
#ifdef __LP64__
__m128i* vv128 = (__m128i*)main_bitvec;
const __m128i* ov128 = (const __m128i*)arg_bitvec;
__m128i* vv128_end = &(vv128[word_ct / 2]);
while (vv128 < vv128_end) {
*vv128 = _mm_or_si128(*ov128++, *vv128);
vv128++;
}
if (word_ct & 1) {
word_ct--;
main_bitvec[word_ct] |= arg_bitvec[word_ct];
}
#else
uintptr_t* vec_end = &(main_bitvec[word_ct]);
while (main_bitvec < vec_end) {
*main_bitvec++ |= *arg_bitvec++;
}
#endif
}
void bitvec_ornot(const uintptr_t* __restrict inverted_or_bitvec, uintptr_t word_ct, uintptr_t* __restrict main_bitvec) {
// main_bitvec := main_bitvec OR (~inverted_or_bitvec)
#ifdef __LP64__
#ifdef __APPLE__
const __m128i all1 = {0xffffffffffffffffLLU, 0xffffffffffffffffLLU};
#else
const __m128i all1 = {-1LL, -1LL};
#endif
__m128i* vv128 = (__m128i*)main_bitvec;
const __m128i* ev128 = (const __m128i*)inverted_or_bitvec;
__m128i* vv128_end = &(vv128[word_ct / 2]);
while (vv128 < vv128_end) {
*vv128 = _mm_or_si128(_mm_xor_si128(*ev128++, all1), *vv128);
vv128++;
}
if (word_ct & 1) {
word_ct--;
main_bitvec[word_ct] |= ~(inverted_or_bitvec[word_ct]);
}
#else
uintptr_t* vec_end = &(main_bitvec[word_ct]);
while (main_bitvec < vec_end) {
*main_bitvec++ |= ~(*inverted_or_bitvec++);
}
#endif
}
void bitvec_xor(const uintptr_t* __restrict arg_bitvec, uintptr_t word_ct, uintptr_t* __restrict main_bitvec) {
// main_bitvec := main_bitvec XOR xor_bitvec
#ifdef __LP64__
__m128i* bitv128 = (__m128i*)main_bitvec;
__m128i* xorv128 = (__m128i*)arg_bitvec;
__m128i* bitv128_end = &(bitv128[word_ct / 2]);
while (bitv128 < bitv128_end) {
*bitv128 = _mm_xor_si128(*xorv128++, *bitv128);
bitv128++;
}
if (word_ct & 1) {
word_ct--;
main_bitvec[word_ct] ^= arg_bitvec[word_ct];
}
#else
uintptr_t* main_bitvec_end = &(main_bitvec[word_ct]);
while (main_bitvec < main_bitvec_end) {
*main_bitvec++ ^= *arg_bitvec++;
}
#endif
}
uint32_t is_monomorphic_a2(const uintptr_t* geno_arr, uint32_t sample_ct) {
const uintptr_t* loop_end = &(geno_arr[sample_ct / BITCT2]);
uint32_t sample_rem = sample_ct % BITCT2;
for (; geno_arr < loop_end; geno_arr++) {
if ((~(*geno_arr)) & FIVEMASK) {
return 0;
}
}
return (sample_rem && ((~(*geno_arr)) & (FIVEMASK >> (BITCT - sample_rem * 2))))? 0 : 1;
}
uint32_t is_monomorphic(const uintptr_t* geno_arr, uint32_t sample_ct) {
uint32_t sample_ctd2 = sample_ct / BITCT2;
uint32_t sample_rem = sample_ct % BITCT2;
uintptr_t ulii;
uintptr_t uljj;
while (sample_ctd2) {
ulii = *geno_arr++;
uljj = (ulii >> 1) & FIVEMASK;
ulii = ~ulii;
// now ulii & FIVEMASK = low bit zero, uljj = high bit one
if (uljj) {
if (uljj & ulii) {
// heterozygote observed
return 0;
}
// homozyg A2 observed
while (1) {
// 00 and 10 now both demonstrate marker is polymorphic
if (ulii & FIVEMASK) {
return 0;
}
if (!(--sample_ctd2)) {
return (sample_rem && ((~(*geno_arr)) & (FIVEMASK >> (BITCT - sample_rem * 2))))? 0 : 1;
}
ulii = ~(*geno_arr++);
}
} else if (ulii & FIVEMASK) {
do {
if (!(--sample_ctd2)) {
return (sample_rem && ((*geno_arr) & (AAAAMASK >> (BITCT - sample_rem * 2))))? 0 : 1;
}
ulii = *geno_arr++;
} while (!(ulii & AAAAMASK));
return 0;
}
sample_ctd2--;
}
if (sample_rem) {
ulii = *geno_arr;
uljj = (ulii >> 1) & FIVEMASK;
ulii = ~ulii;
if ((uljj & ulii) || (uljj && (ulii & (~uljj) & (FIVEMASK >> (BITCT - sample_rem * 2))))) {
return 0;
}
}
return 1;
}
uint32_t less_than_two_genotypes(const uintptr_t* geno_arr, uint32_t sample_ct) {
uint32_t sample_ctd2 = sample_ct / BITCT2;
uint32_t sample_rem = sample_ct % BITCT2;
uintptr_t ulii;
uintptr_t uljj;
uintptr_t ulkk;
uint32_t distinct_genotype_ct;
while (sample_ctd2) {
ulii = *geno_arr++;
uljj = (ulii >> 1) & FIVEMASK;
ulkk = ~ulii;
if (uljj) {
if (uljj & ulii) {
// homozygote major observed; either 00 or 10 now demonstrate marker
// is polymorphic
while (1) {
if (ulkk & FIVEMASK) {
return 0;
}
if (!(--sample_ctd2)) {
return (sample_rem && ((~(*geno_arr)) & (FIVEMASK >> (BITCT - sample_rem * 2))))? 0 : 1;
}
ulkk = ~(*geno_arr++);
}
} else {
// heterozygote observed; either 00 or 11 now means we have 2+
// genotypes
while (1) {
ulii = ~(*geno_arr++);
if (!(--sample_ctd2)) {
return (sample_rem && (((~ulii) ^ (ulii >> 1)) & (FIVEMASK >> (BITCT - sample_rem * 2))))? 0 : 1;
}
if (((~ulii) ^ (ulii >> 1)) & FIVEMASK) {
return 0;
}
}
}
} else if (ulkk & FIVEMASK) {
// homozygous minor observed; either 10 or 11 now demonstrate marker is
// polymorphic
do {
if (!(--sample_ctd2)) {
return (sample_rem && ((*geno_arr) & (AAAAMASK >> (BITCT - sample_rem * 2))))? 0 : 1;
}
ulii = *geno_arr++;
} while (!(ulii & AAAAMASK));
return 0;
}
sample_ctd2--;
}
if (sample_rem) {
ulii = *geno_arr;
uljj = (ulii >> 1) & FIVEMASK;
ulkk = ~ulii;
// homozygous minor present?
distinct_genotype_ct = (ulkk & (~uljj) & (FIVEMASK >> (BITCT - sample_rem * 2)))? 1 : 0;
// heterozygous present?
distinct_genotype_ct += (uljj & ulkk)? 1 : 0;
// homozygous major present?
distinct_genotype_ct += (uljj & ulii)? 1 : 0;
if (distinct_genotype_ct > 1) {
return 0;
}
}
return 1;
}
/*
uint32_t has_three_genotypes(uintptr_t* lptr, uint32_t sample_ct) {
uintptr_t* lptr_end = &(lptr[sample_ct / BITCT2]);
uint32_t sample_rem = sample_ct % BITCT2;
uintptr_t* cur_lptr;
uintptr_t ulii;
uintptr_t uljj;
cur_lptr = lptr;
while (1) {
ulii = ~(*cur_lptr);
uljj = ulii & (ulii >> 1) & FIVEMASK;
if (cur_lptr == lptr_end) {
if ((!sample_rem) || (!(uljj << (BITCT - sample_rem * 2)))) {
return 0;
}
break;
}
if (uljj) {
// found hom A1
break;
}
cur_lptr++;
}
cur_lptr = lptr;
// zero-padding is benign for het and hom A2 checks
lptr_end = &(lptr[QUATERCT_TO_WORDCT(sample_ct)]);
while (1) {
ulii = *cur_lptr;
uljj = (ulii >> 1) & FIVEMASK;
if ((~ulii) & uljj) {
break;
}
if (++cur_lptr == lptr_end) {
return 0;
}
}
cur_lptr = lptr;
do {
ulii = *cur_lptr;
uljj = (ulii >> 1) & FIVEMASK;
if (ulii & uljj) {
return 1;
}
} while (++cur_lptr < lptr_end);
return 0;
}
*/
#ifdef __LP64__
// Basic SSE2 implementation of Lauradoux/Walisch popcount.
static inline uintptr_t popcount_vecs(const __m128i* vptr, uintptr_t ct) {
// popcounts vptr[0..(ct-1)]. Assumes ct is a multiple of 3 (0 ok).
const __m128i m1 = {FIVEMASK, FIVEMASK};
const __m128i m2 = {0x3333333333333333LLU, 0x3333333333333333LLU};
const __m128i m4 = {0x0f0f0f0f0f0f0f0fLLU, 0x0f0f0f0f0f0f0f0fLLU};
const __m128i m8 = {0x00ff00ff00ff00ffLLU, 0x00ff00ff00ff00ffLLU};
uintptr_t tot = 0;
const __m128i* vend;
__m128i count1;
__m128i count2;
__m128i half1;
__m128i half2;
__univec acc;
while (ct >= 30) {
ct -= 30;
vend = &(vptr[30]);
popcount_vecs_main_loop:
acc.vi = _mm_setzero_si128();
do {
count1 = *vptr++;
count2 = *vptr++;
half1 = *vptr++;
half2 = _mm_and_si128(_mm_srli_epi64(half1, 1), m1);
half1 = _mm_and_si128(half1, m1);
// Two bits can represent values from 0-3, so make each pair in count1
// count2 store a partial bitcount covering themselves AND another bit
// from elsewhere.
count1 = _mm_sub_epi64(count1, _mm_and_si128(_mm_srli_epi64(count1, 1), m1));
count2 = _mm_sub_epi64(count2, _mm_and_si128(_mm_srli_epi64(count2, 1), m1));
count1 = _mm_add_epi64(count1, half1);
count2 = _mm_add_epi64(count2, half2);
// Four bits represent 0-15, so we can safely add four 0-3 partial
// bitcounts together.
count1 = _mm_add_epi64(_mm_and_si128(count1, m2), _mm_and_si128(_mm_srli_epi64(count1, 2), m2));
count1 = _mm_add_epi64(count1, _mm_add_epi64(_mm_and_si128(count2, m2), _mm_and_si128(_mm_srli_epi64(count2, 2), m2)));
// Accumulator stores sixteen 0-255 counts in parallel.
acc.vi = _mm_add_epi64(acc.vi, _mm_add_epi64(_mm_and_si128(count1, m4), _mm_and_si128(_mm_srli_epi64(count1, 4), m4)));
} while (vptr < vend);
acc.vi = _mm_add_epi64(_mm_and_si128(acc.vi, m8), _mm_and_si128(_mm_srli_epi64(acc.vi, 8), m8));
tot += ((acc.u8[0] + acc.u8[1]) * 0x1000100010001LLU) >> 48;
}
if (ct) {
vend = &(vptr[ct]);
ct = 0;
goto popcount_vecs_main_loop;
}
return tot;
}
static inline uintptr_t popcount2_vecs(const __m128i* vptr, uintptr_t ct) {
// assumes ct is a multiple of 6.
const __m128i m2 = {0x3333333333333333LLU, 0x3333333333333333LLU};
const __m128i m4 = {0x0f0f0f0f0f0f0f0fLLU, 0x0f0f0f0f0f0f0f0fLLU};
const __m128i m8 = {0x00ff00ff00ff00ffLLU, 0x00ff00ff00ff00ffLLU};
uintptr_t tot = 0;
const __m128i* vend;
__m128i loader1;
__m128i loader2;
__m128i count1;
__m128i count2;
__univec acc;
while (ct >= 30) {
ct -= 30;
vend = &(vptr[30]);
popcount2_vecs_main_loop:
acc.vi = _mm_setzero_si128();
do {
loader1 = *vptr++;
loader2 = *vptr++;
count1 = _mm_add_epi64(_mm_and_si128(loader1, m2), _mm_and_si128(_mm_srli_epi64(loader1, 2), m2));
count2 = _mm_add_epi64(_mm_and_si128(loader2, m2), _mm_and_si128(_mm_srli_epi64(loader2, 2), m2));
loader1 = *vptr++;
loader2 = *vptr++;
count1 = _mm_add_epi64(count1, _mm_add_epi64(_mm_and_si128(loader1, m2), _mm_and_si128(_mm_srli_epi64(loader1, 2), m2)));
count2 = _mm_add_epi64(count2, _mm_add_epi64(_mm_and_si128(loader2, m2), _mm_and_si128(_mm_srli_epi64(loader2, 2), m2)));
loader1 = *vptr++;
loader2 = *vptr++;
count1 = _mm_add_epi64(count1, _mm_add_epi64(_mm_and_si128(loader1, m2), _mm_and_si128(_mm_srli_epi64(loader1, 2), m2)));
count2 = _mm_add_epi64(count2, _mm_add_epi64(_mm_and_si128(loader2, m2), _mm_and_si128(_mm_srli_epi64(loader2, 2), m2)));
acc.vi = _mm_add_epi64(acc.vi, _mm_add_epi64(_mm_and_si128(count1, m4), _mm_and_si128(_mm_srli_epi64(count1, 4), m4)));
acc.vi = _mm_add_epi64(acc.vi, _mm_add_epi64(_mm_and_si128(count2, m4), _mm_and_si128(_mm_srli_epi64(count2, 4), m4)));
} while (vptr < vend);
acc.vi = _mm_add_epi64(_mm_and_si128(acc.vi, m8), _mm_and_si128(_mm_srli_epi64(acc.vi, 8), m8));
tot += ((acc.u8[0] + acc.u8[1]) * 0x1000100010001LLU) >> 48;
}
if (ct) {
vend = &(vptr[ct]);
ct = 0;
goto popcount2_vecs_main_loop;
}
return tot;
}
static inline uintptr_t popcount_vecs_exclude(const __m128i* __restrict vptr, const __m128i* __restrict exclude_ptr, uintptr_t ct) {
// popcounts vptr ANDNOT exclude_ptr[0..(ct-1)]. ct is a multiple of 3.
const __m128i m1 = {FIVEMASK, FIVEMASK};
const __m128i m2 = {0x3333333333333333LLU, 0x3333333333333333LLU};
const __m128i m4 = {0x0f0f0f0f0f0f0f0fLLU, 0x0f0f0f0f0f0f0f0fLLU};
const __m128i m8 = {0x00ff00ff00ff00ffLLU, 0x00ff00ff00ff00ffLLU};
uintptr_t tot = 0;
const __m128i* vend;
__m128i count1, count2, half1, half2;
__univec acc;
while (ct >= 30) {
ct -= 30;
vend = &(vptr[30]);
popcount_vecs_exclude_main_loop:
acc.vi = _mm_setzero_si128();
do {
// nots the FIRST value
count1 = _mm_andnot_si128(*exclude_ptr++, *vptr++);
count2 = _mm_andnot_si128(*exclude_ptr++, *vptr++);
half1 = _mm_andnot_si128(*exclude_ptr++, *vptr++);
half2 = _mm_and_si128(_mm_srli_epi64(half1, 1), m1);
half1 = _mm_and_si128(half1, m1);
count1 = _mm_sub_epi64(count1, _mm_and_si128(_mm_srli_epi64(count1, 1), m1));
count2 = _mm_sub_epi64(count2, _mm_and_si128(_mm_srli_epi64(count2, 1), m1));
count1 = _mm_add_epi64(count1, half1);
count2 = _mm_add_epi64(count2, half2);
count1 = _mm_add_epi64(_mm_and_si128(count1, m2), _mm_and_si128(_mm_srli_epi64(count1, 2), m2));
count1 = _mm_add_epi64(count1, _mm_add_epi64(_mm_and_si128(count2, m2), _mm_and_si128(_mm_srli_epi64(count2, 2), m2)));
acc.vi = _mm_add_epi64(acc.vi, _mm_add_epi64(_mm_and_si128(count1, m4), _mm_and_si128(_mm_srli_epi64(count1, 4), m4)));
} while (vptr < vend);
acc.vi = _mm_add_epi64(_mm_and_si128(acc.vi, m8), _mm_and_si128(_mm_srli_epi64(acc.vi, 8), m8));
tot += ((acc.u8[0] + acc.u8[1]) * 0x1000100010001LLU) >> 48;
}
if (ct) {
vend = &(vptr[ct]);
ct = 0;
goto popcount_vecs_exclude_main_loop;
}
return tot;
}
static inline uintptr_t popcount_vecs_intersect(const __m128i* __restrict vptr1, const __m128i* __restrict vptr2, uintptr_t ct) {
// popcounts vptr1 AND vptr2[0..(ct-1)]. ct is a multiple of 3.
const __m128i m1 = {FIVEMASK, FIVEMASK};
const __m128i m2 = {0x3333333333333333LLU, 0x3333333333333333LLU};
const __m128i m4 = {0x0f0f0f0f0f0f0f0fLLU, 0x0f0f0f0f0f0f0f0fLLU};
const __m128i m8 = {0x00ff00ff00ff00ffLLU, 0x00ff00ff00ff00ffLLU};
uintptr_t tot = 0;
const __m128i* vend1;
__m128i count1, count2, half1, half2;
__univec acc;
while (ct >= 30) {
ct -= 30;
vend1 = &(vptr1[30]);
popcount_vecs_intersect_main_loop:
acc.vi = _mm_setzero_si128();
do {
count1 = _mm_and_si128(*vptr2++, *vptr1++);
count2 = _mm_and_si128(*vptr2++, *vptr1++);
half1 = _mm_and_si128(*vptr2++, *vptr1++);
half2 = _mm_and_si128(_mm_srli_epi64(half1, 1), m1);
half1 = _mm_and_si128(half1, m1);
count1 = _mm_sub_epi64(count1, _mm_and_si128(_mm_srli_epi64(count1, 1), m1));
count2 = _mm_sub_epi64(count2, _mm_and_si128(_mm_srli_epi64(count2, 1), m1));
count1 = _mm_add_epi64(count1, half1);
count2 = _mm_add_epi64(count2, half2);
count1 = _mm_add_epi64(_mm_and_si128(count1, m2), _mm_and_si128(_mm_srli_epi64(count1, 2), m2));
count1 = _mm_add_epi64(count1, _mm_add_epi64(_mm_and_si128(count2, m2), _mm_and_si128(_mm_srli_epi64(count2, 2), m2)));
acc.vi = _mm_add_epi64(acc.vi, _mm_add_epi64(_mm_and_si128(count1, m4), _mm_and_si128(_mm_srli_epi64(count1, 4), m4)));
} while (vptr1 < vend1);
acc.vi = _mm_add_epi64(_mm_and_si128(acc.vi, m8), _mm_and_si128(_mm_srli_epi64(acc.vi, 8), m8));
tot += ((acc.u8[0] + acc.u8[1]) * 0x1000100010001LLU) >> 48;
}
if (ct) {
vend1 = &(vptr1[ct]);
ct = 0;
goto popcount_vecs_intersect_main_loop;
}
return tot;
}
#endif
uintptr_t popcount_longs(const uintptr_t* lptr, uintptr_t word_ct) {
// Efficiently popcounts lptr[0..(word_ct - 1)]. In the 64-bit case, lptr[]
// must be 16-byte aligned.
// The popcount_longs_nzbase() wrapper takes care of starting from a later
// index.
uintptr_t tot = 0;
const uintptr_t* lptr_end = &(lptr[word_ct]);
#ifdef __LP64__
uintptr_t six_ct;
const __m128i* vptr;
vptr = (const __m128i*)lptr;
six_ct = word_ct / 6;
tot += popcount_vecs(vptr, six_ct * 3);
lptr = &(lptr[six_ct * 6]);
#else
// The humble 16-bit lookup table actually beats
// http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
// on my development machine by a hair.
// However, if we take the hint from Lauradoux/Walisch and postpone the
// multiply and right shift, this is no longer true. Ah well.
const uintptr_t* lptr_six_end;
uintptr_t tmp_stor;
uintptr_t loader;
uintptr_t ulii;
uintptr_t uljj;
lptr_six_end = &(lptr[word_ct - (word_ct % 6)]);
while (lptr < lptr_six_end) {
loader = *lptr++;
ulii = loader - ((loader >> 1) & FIVEMASK);
loader = *lptr++;
uljj = loader - ((loader >> 1) & FIVEMASK);
loader = *lptr++;
ulii += (loader >> 1) & FIVEMASK;
uljj += loader & FIVEMASK;
ulii = (ulii & 0x33333333) + ((ulii >> 2) & 0x33333333);
ulii += (uljj & 0x33333333) + ((uljj >> 2) & 0x33333333);
tmp_stor = (ulii & 0x0f0f0f0f) + ((ulii >> 4) & 0x0f0f0f0f);
loader = *lptr++;
ulii = loader - ((loader >> 1) & FIVEMASK);
loader = *lptr++;
uljj = loader - ((loader >> 1) & FIVEMASK);
loader = *lptr++;
ulii += (loader >> 1) & FIVEMASK;
uljj += loader & FIVEMASK;
ulii = (ulii & 0x33333333) + ((ulii >> 2) & 0x33333333);
ulii += (uljj & 0x33333333) + ((uljj >> 2) & 0x33333333);
tmp_stor += (ulii & 0x0f0f0f0f) + ((ulii >> 4) & 0x0f0f0f0f);
// Each 8-bit slot stores a number in 0..48. Multiplying by 0x01010101 is
// equivalent to the left-shifts and adds we need to sum those four 8-bit
// numbers in the high-order slot.
tot += (tmp_stor * 0x01010101) >> 24;
}
#endif
while (lptr < lptr_end) {
tot += popcount_long(*lptr++);
}
return tot;
}
uintptr_t popcount2_longs(const uintptr_t* lptr, uintptr_t word_ct) {
// treats lptr[] as an array of two-bit instead of one-bit numbers
uintptr_t tot = 0;
const uintptr_t* lptr_end = &(lptr[word_ct]);
#ifdef __LP64__
uintptr_t twelve_ct;
const __m128i* vptr;
vptr = (const __m128i*)lptr;
twelve_ct = word_ct / 12;
tot += popcount2_vecs(vptr, twelve_ct * 6);
lptr = &(lptr[twelve_ct * 12]);
#else
const uintptr_t* lptr_six_end;
uintptr_t loader1;
uintptr_t loader2;
uintptr_t ulii;
uintptr_t uljj;
lptr_six_end = &(lptr[word_ct - (word_ct % 6)]);
while (lptr < lptr_six_end) {
loader1 = *lptr++;
loader2 = *lptr++;
ulii = (loader1 & 0x33333333) + ((loader1 >> 2) & 0x33333333);
uljj = (loader2 & 0x33333333) + ((loader2 >> 2) & 0x33333333);
loader1 = *lptr++;
loader2 = *lptr++;
ulii += (loader1 & 0x33333333) + ((loader1 >> 2) & 0x33333333);
uljj += (loader2 & 0x33333333) + ((loader2 >> 2) & 0x33333333);
loader1 = *lptr++;
loader2 = *lptr++;
ulii += (loader1 & 0x33333333) + ((loader1 >> 2) & 0x33333333);
uljj += (loader2 & 0x33333333) + ((loader2 >> 2) & 0x33333333);
ulii = (ulii & 0x0f0f0f0f) + ((ulii >> 4) & 0x0f0f0f0f);
ulii += (uljj & 0x0f0f0f0f) + ((uljj >> 4) & 0x0f0f0f0f);
// Each 8-bit slot stores a number in 0..48. Multiplying by 0x01010101 is
// equivalent to the left-shifts and adds we need to sum those four 8-bit
// numbers in the high-order slot.
tot += (ulii * 0x01010101) >> 24;
}
#endif
while (lptr < lptr_end) {
tot += popcount2_long(*lptr++);
}
return tot;
}
uintptr_t popcount_bit_idx(const uintptr_t* lptr, uintptr_t start_idx, uintptr_t end_idx) {
uintptr_t start_idxl = start_idx / BITCT;
uintptr_t start_idxlr = start_idx & (BITCT - 1);
uintptr_t end_idxl = end_idx / BITCT;
uintptr_t end_idxlr = end_idx & (BITCT - 1);
uintptr_t ct = 0;
if (start_idxl == end_idxl) {
return popcount_long(lptr[start_idxl] & ((ONELU << end_idxlr) - (ONELU << start_idxlr)));
}
if (start_idxlr) {
ct = popcount_long(lptr[start_idxl++] >> start_idxlr);
}
if (end_idxl > start_idxl) {
ct += popcount_longs_nzbase(lptr, start_idxl, end_idxl);
}
if (end_idxlr) {
ct += popcount_long(lptr[end_idxl] & ((ONELU << end_idxlr) - ONELU));
}
return ct;
}
uint32_t chrom_window_max(const uint32_t* marker_pos, const uintptr_t* marker_exclude, const Chrom_info* chrom_info_ptr, uint32_t chrom_idx, uint32_t ct_max, uint32_t bp_max, uint32_t cur_window_max) {
// okay, it's absurd to keep rewriting this from scratch, especially given
// that makes it likely that some reimplementations suck (--indep{-pairwise}
// version was O(n^2) instead of O(n); sure, it didn't really matter because
// the main calculation was more expensive, but still, ugh).
if (cur_window_max >= ct_max) {
return ct_max;
}
// assumes chrom_idx exists
uint32_t chrom_fo_idx = chrom_info_ptr->chrom_idx_to_foidx[chrom_idx];
uint32_t chrom_end = chrom_info_ptr->chrom_fo_vidx_start[chrom_fo_idx + 1];
uint32_t marker_uidx = next_unset(marker_exclude, chrom_info_ptr->chrom_fo_vidx_start[chrom_fo_idx], chrom_end);
uint32_t marker_ct = chrom_end - marker_uidx - popcount_bit_idx(marker_exclude, marker_uidx, chrom_end);
if (marker_ct <= cur_window_max) {
return cur_window_max;
}
uint32_t window_idx_first = 0;
uint32_t window_uidx_first = marker_uidx;
uint32_t window_pos_first = marker_pos[marker_uidx];
uint32_t marker_idx;
uint32_t marker_pos_thresh;
for (marker_idx = 0; marker_idx < marker_ct; marker_uidx++, marker_idx++) {
next_unset_unsafe_ck(marker_exclude, &marker_uidx);
marker_pos_thresh = marker_pos[marker_uidx];
if (marker_pos_thresh < bp_max) {
marker_pos_thresh = 0;
} else {
marker_pos_thresh -= bp_max;
}
if (marker_pos_thresh > window_pos_first) {
do {
window_uidx_first++;
next_unset_unsafe_ck(marker_exclude, &window_uidx_first);
window_pos_first = marker_pos[window_uidx_first];
window_idx_first++;
} while (marker_pos_thresh > window_pos_first);
} else if (marker_idx - window_idx_first == cur_window_max) {
if (++cur_window_max == ct_max) {
return cur_window_max;
}
}
}
return cur_window_max;
}
uint32_t window_back(const uint32_t* __restrict marker_pos, const double* __restrict marker_cms, const uintptr_t* marker_exclude, uint32_t marker_uidx_min, uint32_t marker_uidx_start, uint32_t count_max, uint32_t bp_max, double cm_max, uint32_t* __restrict window_trail_ct_ptr) {
// Finds the earliest location which is within count_max sites, bp_max bps,
// and (if marker_cms != nullptr) cm_max centimorgans.
// count_max must be positive.
if (marker_uidx_min == marker_uidx_start) {
// special-case this since it happens frequently
*window_trail_ct_ptr = 0;
return marker_uidx_min;
}
double min_cm = marker_cms? (marker_cms[marker_uidx_start] - cm_max) : 0.0;
uint32_t min_pos = 0;
uint32_t marker_uwidx_cur = marker_uidx_start / BITCT;
uint32_t uii = marker_uidx_start % BITCT;
uint32_t marker_uidx_last = marker_uidx_start;
uint32_t remaining_count = count_max;
const uintptr_t* marker_exclude_cur = &(marker_exclude[marker_uwidx_cur]);
uintptr_t cur_word;
uint32_t ujj;
uint32_t ukk;
marker_uwidx_cur *= BITCT;
if (bp_max <= marker_pos[marker_uidx_start]) {
min_pos = marker_pos[marker_uidx_start] - bp_max;
}
if (!uii) {
goto window_back_zstart;
}
cur_word = (~(*marker_exclude_cur)) & ((ONELU << uii) - ONELU);
while (1) {
if (marker_uwidx_cur <= marker_uidx_min) {
cur_word &= ~((ONELU << (marker_uidx_min % BITCT)) - ONELU);
marker_uwidx_cur = marker_uidx_min;
uii = popcount_long(cur_word);
if (uii >= remaining_count) {
goto window_back_count;
} else if ((marker_pos[marker_uwidx_cur] < min_pos) || (marker_cms && (marker_cms[marker_uwidx_cur] < min_cm))) {
goto window_back_find_first_pos;
} else {
goto window_back_min;
}
}
uii = popcount_long(cur_word);
if (uii >= remaining_count) {
window_back_count:
uii -= remaining_count; // now a count of number of bits to advance
while (uii) {
cur_word &= cur_word - 1;
uii--;
}
// bugfix (7 May 2017): forgot to round marker_uwidx_cur down to word
// boundary, before adding CTZLU(cur_word) offset
marker_uwidx_cur = (marker_uwidx_cur & (~(BITCT - ONELU))) + CTZLU(cur_word);
if ((marker_pos[marker_uwidx_cur] < min_pos) || (marker_cms && (marker_cms[marker_uwidx_cur] < min_cm))) {
goto window_back_find_first_pos;
}
*window_trail_ct_ptr = count_max;
return marker_uwidx_cur;
}
if ((marker_pos[marker_uwidx_cur] < min_pos) || (marker_cms && (marker_cms[marker_uwidx_cur] < min_cm))) {
window_back_find_first_pos:
ujj = uint32arr_greater_than(&(marker_pos[marker_uwidx_cur]), marker_uidx_last - marker_uwidx_cur, min_pos);
if (marker_cms) {
ukk = doublearr_greater_than(&(marker_cms[marker_uwidx_cur]), marker_uidx_last - marker_uwidx_cur, min_cm);
if (ujj < ukk) {
ujj = ukk;
}
}
marker_uwidx_cur += ujj;
if (marker_uwidx_cur > marker_uidx_min) {
next_unset_unsafe_ck(marker_exclude, &marker_uwidx_cur);
}
window_back_min:
*window_trail_ct_ptr = marker_uidx_start - marker_uwidx_cur - popcount_bit_idx(marker_exclude, marker_uwidx_cur, marker_uidx_start);
return marker_uwidx_cur;
}
remaining_count -= uii;
marker_uidx_last = marker_uwidx_cur;
window_back_zstart:
cur_word = ~(*(--marker_exclude_cur));
marker_uwidx_cur -= BITCT;
}
}
uint32_t window_forward(const uint32_t* __restrict marker_pos, const double* __restrict marker_cms, const uintptr_t* marker_exclude, uint32_t marker_uidx_start, uint32_t marker_uidx_last, uint32_t count_max, uint32_t bp_max, double cm_max, uint32_t* __restrict window_lead_ct_ptr) {
// window_lead_ct_ptr currently cannot be nullptr
if (marker_uidx_start == marker_uidx_last) {
*window_lead_ct_ptr = 0;
return marker_uidx_start;
}
double max_cm = marker_cms? (cm_max + marker_cms[marker_uidx_start]) : 0.0;
uint32_t marker_uwidx_prev = marker_uidx_start;
uint32_t max_pos = bp_max + marker_pos[marker_uidx_start];
uint32_t marker_uwidx_cur = (marker_uidx_start + 1) / BITCT;
uint32_t uii = (marker_uidx_start + 1) % BITCT;
uint32_t remaining_count = count_max;
const uintptr_t* marker_exclude_cur = &(marker_exclude[marker_uwidx_cur]);
uintptr_t cur_word;
uint32_t ujj;
uint32_t ukk;
marker_uwidx_cur *= BITCT;
cur_word = ~((*marker_exclude_cur) | ((ONELU << uii) - ONELU));
while (1) {
uii = popcount_long(cur_word);
if (uii >= remaining_count) {
while (--remaining_count) {
cur_word &= cur_word - 1;
}
marker_uwidx_cur += CTZLU(cur_word);
if (marker_uwidx_cur <= marker_uidx_last) {
if ((marker_pos[marker_uwidx_cur] > max_pos) || (marker_cms && (marker_cms[marker_uwidx_cur] > max_cm))) {
break;
}
*window_lead_ct_ptr = count_max;
return marker_uwidx_cur;
}
if ((marker_pos[marker_uidx_last] <= max_pos) && ((!marker_cms) || (marker_cms[marker_uidx_last] <= max_cm))) {
marker_uwidx_prev = marker_uidx_last;
goto window_forward_return;
}
marker_uwidx_cur = marker_uidx_last;
break;
}
marker_uwidx_cur += BITCT;
if (marker_uwidx_cur >= marker_uidx_last) {
if ((marker_pos[marker_uidx_last] <= max_pos) && ((!marker_cms) || (marker_cms[marker_uidx_last] <= max_cm))) {
marker_uwidx_prev = marker_uidx_last;
goto window_forward_return;
}
marker_uwidx_cur = marker_uidx_last;
break;
} else if ((marker_pos[marker_uwidx_cur] > max_pos) || (marker_cms && (marker_cms[marker_uwidx_cur] > max_cm))) {
break;
}
marker_uwidx_prev = marker_uwidx_cur;
remaining_count -= uii;
cur_word = ~(*(++marker_exclude_cur));
}
ujj = uint32arr_greater_than(&(marker_pos[marker_uwidx_prev]), marker_uwidx_cur - marker_uwidx_prev, max_pos + 1);
if (marker_cms) {
ukk = doublearr_greater_than(&(marker_cms[marker_uwidx_prev]), marker_uwidx_cur - marker_uwidx_prev, max_cm * (1 + SMALL_EPSILON));
if (ujj > ukk) {
ujj = ukk;
}
}
marker_uwidx_prev += ujj;
prev_unset_unsafe_ck(marker_exclude, &marker_uwidx_prev);
window_forward_return:
*window_lead_ct_ptr = marker_uwidx_prev - marker_uidx_start - popcount_bit_idx(marker_exclude, marker_uidx_start, marker_uwidx_prev);
return marker_uwidx_prev;
}
uintptr_t jump_forward_unset_unsafe(const uintptr_t* bitvec, uintptr_t cur_pos, uintptr_t forward_ct) {
// advances forward_ct unset bits; forward_ct must be positive. (stays put
// if forward_ct == 1 and current bit is unset. may want to tweak this
// interface, easy to introduce off-by-one bugs...)
// In usual 64-bit case, also assumes bitvec is 16-byte aligned and the end
// of the trailing 16-byte block can be safely read from.
uintptr_t widx = cur_pos / BITCT;
uintptr_t ulii = cur_pos % BITCT;
const uintptr_t* bptr = &(bitvec[widx]);
uintptr_t uljj;
uintptr_t ulkk;
#ifdef __LP64__
const __m128i* vptr;
#endif
if (ulii) {
uljj = (~(*bptr)) >> ulii;
ulkk = popcount_long(uljj);
if (ulkk >= forward_ct) {
jump_forward_unset_unsafe_finish:
ulkk = CTZLU(uljj);
while (--forward_ct) {
uljj &= uljj - 1;
ulkk = CTZLU(uljj);
}
return widx * BITCT + ulii + ulkk;
}
forward_ct -= ulkk;
widx++;
bptr++;
}
ulii = 0;
#ifdef __LP64__
if (widx & 1) {
uljj = ~(*bptr);
ulkk = popcount_long(uljj);
if (ulkk >= forward_ct) {
goto jump_forward_unset_unsafe_finish;
}
forward_ct -= ulkk;
bptr++;
}
vptr = (const __m128i*)bptr;
while (forward_ct > BITCT * 6) {
uljj = ((forward_ct - 1) / (BITCT * 6)) * 3;
ulkk = popcount_vecs(vptr, uljj);
vptr = &(vptr[uljj]);
forward_ct -= uljj * BITCT * 2 - ulkk;
}
bptr = (const uintptr_t*)vptr;
while (forward_ct > BITCT) {
forward_ct -= popcount_long(~(*bptr++));
}
#else
while (forward_ct > BITCT) {
uljj = (forward_ct - 1) / BITCT;
ulkk = popcount_longs(bptr, uljj);
bptr = &(bptr[uljj]);
forward_ct -= uljj * BITCT - ulkk;
}
#endif
while (1) {
uljj = ~(*bptr);
ulkk = popcount_long(uljj);
if (ulkk >= forward_ct) {
widx = (uintptr_t)(bptr - bitvec);
goto jump_forward_unset_unsafe_finish;
}
forward_ct -= ulkk;
bptr++;
}
}
uintptr_t popcount_longs_exclude(const uintptr_t* __restrict lptr, const uintptr_t* __restrict exclude_arr, uintptr_t end_idx) {
// popcounts lptr ANDNOT exclude_arr[0..(end_idx-1)].
// N.B. on 64-bit systems, assumes lptr and exclude_arr are 16-byte aligned.
uintptr_t tot = 0;
const uintptr_t* lptr_end = &(lptr[end_idx]);
#ifdef __LP64__
uintptr_t six_ct = end_idx / 6;
tot += popcount_vecs_exclude((const __m128i*)lptr, (const __m128i*)exclude_arr, six_ct * 3);
lptr = &(lptr[six_ct * 6]);
exclude_arr = &(exclude_arr[six_ct * 6]);
#else
const uintptr_t* lptr_six_end;
uintptr_t tmp_stor;
uintptr_t loader;
uintptr_t ulii;
uintptr_t uljj;
lptr_six_end = &(lptr[end_idx - (end_idx % 6)]);
while (lptr < lptr_six_end) {
loader = (*lptr++) & (~(*exclude_arr++));
ulii = loader - ((loader >> 1) & FIVEMASK);
loader = (*lptr++) & (~(*exclude_arr++));
uljj = loader - ((loader >> 1) & FIVEMASK);
loader = (*lptr++) & (~(*exclude_arr++));
ulii += (loader >> 1) & FIVEMASK;
uljj += loader & FIVEMASK;
ulii = (ulii & 0x33333333) + ((ulii >> 2) & 0x33333333);
ulii += (uljj & 0x33333333) + ((uljj >> 2) & 0x33333333);
tmp_stor = (ulii & 0x0f0f0f0f) + ((ulii >> 4) & 0x0f0f0f0f);
loader = (*lptr++) & (~(*exclude_arr++));
ulii = loader - ((loader >> 1) & FIVEMASK);
loader = (*lptr++) & (~(*exclude_arr++));
uljj = loader - ((loader >> 1) & FIVEMASK);
loader = (*lptr++) & (~(*exclude_arr++));
ulii += (loader >> 1) & FIVEMASK;
uljj += loader & FIVEMASK;
ulii = (ulii & 0x33333333) + ((ulii >> 2) & 0x33333333);
ulii += (uljj & 0x33333333) + ((uljj >> 2) & 0x33333333);
tmp_stor += (ulii & 0x0f0f0f0f) + ((ulii >> 4) & 0x0f0f0f0f);
// Each 8-bit slot stores a number in 0..48. Multiplying by 0x01010101 is
// equivalent to the left-shifts and adds we need to sum those four 8-bit
// numbers in the high-order slot.
tot += (tmp_stor * 0x01010101) >> 24;
}
#endif
while (lptr < lptr_end) {
tot += popcount_long((*lptr++) & (~(*exclude_arr++)));
}
return tot;
}
uintptr_t popcount_longs_intersect(const uintptr_t* __restrict lptr1, const uintptr_t* __restrict lptr2, uintptr_t word_ct) {
uintptr_t tot = 0;
const uintptr_t* lptr1_end = &(lptr1[word_ct]);
#ifdef __LP64__
uintptr_t six_ct = word_ct / 6;
tot += popcount_vecs_intersect((const __m128i*)lptr1, (const __m128i*)lptr2, six_ct * 3);
lptr1 = &(lptr1[six_ct * 6]);
lptr2 = &(lptr2[six_ct * 6]);
#else
const uintptr_t* lptr1_six_end;
uintptr_t tmp_stor;
uintptr_t loader;
uintptr_t ulii;
uintptr_t uljj;
lptr1_six_end = &(lptr1[word_ct - (word_ct % 6)]);
while (lptr1 < lptr1_six_end) {
loader = (*lptr1++) & (*lptr2++);
ulii = loader - ((loader >> 1) & FIVEMASK);
loader = (*lptr1++) & (*lptr2++);
uljj = loader - ((loader >> 1) & FIVEMASK);
loader = (*lptr1++) & (*lptr2++);
ulii += (loader >> 1) & FIVEMASK;
uljj += loader & FIVEMASK;
ulii = (ulii & 0x33333333) + ((ulii >> 2) & 0x33333333);
ulii += (uljj & 0x33333333) + ((uljj >> 2) & 0x33333333);
tmp_stor = (ulii & 0x0f0f0f0f) + ((ulii >> 4) & 0x0f0f0f0f);
loader = (*lptr1++) & (*lptr2++);
ulii = loader - ((loader >> 1) & FIVEMASK);
loader = (*lptr1++) & (*lptr2++);
uljj = loader - ((loader >> 1) & FIVEMASK);
loader = (*lptr1++) & (*lptr2++);
ulii += (loader >> 1) & FIVEMASK;
uljj += loader & FIVEMASK;
ulii = (ulii & 0x33333333) + ((ulii >> 2) & 0x33333333);
ulii += (uljj & 0x33333333) + ((uljj >> 2) & 0x33333333);
tmp_stor += (ulii & 0x0f0f0f0f) + ((ulii >> 4) & 0x0f0f0f0f);
// Each 8-bit slot stores a number in 0..48. Multiplying by 0x01010101 is
// equivalent to the left-shifts and adds we need to sum those four 8-bit
// numbers in the high-order slot.
tot += (tmp_stor * 0x01010101) >> 24;
}
#endif
while (lptr1 < lptr1_end) {
tot += popcount_long((*lptr1++) & (*lptr2++));
}
return tot;
}
void vertical_bitct_subtract(const uintptr_t* bitarr, uint32_t item_ct, uint32_t* sum_arr) {
// assumes trailing bits are zeroed out
uintptr_t cur_word;
uint32_t idx_offset;
uint32_t last_set_bit;
for (idx_offset = 0; idx_offset < item_ct; idx_offset += BITCT) {
cur_word = *bitarr++;
while (cur_word) {
last_set_bit = CTZLU(cur_word);
sum_arr[idx_offset + last_set_bit] -= 1;
cur_word &= cur_word - ONELU;
}
}
}
#ifdef __LP64__
void count_2freq_dbl_960b(const VECITYPE* geno_vvec, const VECITYPE* geno_vvec_end, const VECITYPE* __restrict mask1vp, const VECITYPE* __restrict mask2vp, uint32_t* __restrict ct1abp, uint32_t* __restrict ct1cp, uint32_t* __restrict ct2abp, uint32_t* __restrict ct2cp) {
const __m128i m2 = {0x3333333333333333LLU, 0x3333333333333333LLU};
const __m128i m4 = {0x0f0f0f0f0f0f0f0fLLU, 0x0f0f0f0f0f0f0f0fLLU};
__m128i loader;
__m128i loader2;
__m128i loader3;
__m128i to_ct1_ab;
__m128i to_ct_abtmp;
__m128i to_ct1_c;
__m128i to_ct2_ab;
__m128i to_ct2_c;
__univec acc1_ab;
__univec acc1_c;
__univec acc2_ab;
__univec acc2_c;
acc1_ab.vi = _mm_setzero_si128();
acc1_c.vi = _mm_setzero_si128();
acc2_ab.vi = _mm_setzero_si128();
acc2_c.vi = _mm_setzero_si128();
do {
loader = *geno_vvec++;
loader2 = *mask1vp++;
loader3 = _mm_and_si128(loader2, _mm_srli_epi64(loader, 1));
loader2 = _mm_and_si128(loader2, loader);
to_ct1_ab = _mm_add_epi64(loader3, loader2);
to_ct1_c = _mm_andnot_si128(loader3, loader2);
loader2 = *mask2vp++;
loader3 = _mm_and_si128(loader2, _mm_srli_epi64(loader, 1));
loader2 = _mm_and_si128(loader2, loader);
to_ct2_ab = _mm_add_epi64(loader3, loader2);
to_ct2_c = _mm_andnot_si128(loader3, loader2);
to_ct1_ab = _mm_add_epi64(_mm_and_si128(to_ct1_ab, m2), _mm_and_si128(_mm_srli_epi64(to_ct1_ab, 2), m2));
to_ct2_ab = _mm_add_epi64(_mm_and_si128(to_ct2_ab, m2), _mm_and_si128(_mm_srli_epi64(to_ct2_ab, 2), m2));
loader = *geno_vvec++;
loader2 = *mask1vp++;
loader3 = _mm_and_si128(loader2, _mm_srli_epi64(loader, 1));
loader2 = _mm_and_si128(loader2, loader);
to_ct_abtmp = _mm_add_epi64(loader3, loader2);
to_ct1_c = _mm_add_epi64(to_ct1_c, _mm_andnot_si128(loader3, loader2));
to_ct1_ab = _mm_add_epi64(to_ct1_ab, _mm_add_epi64(_mm_and_si128(to_ct_abtmp, m2), _mm_and_si128(_mm_srli_epi64(to_ct_abtmp, 2), m2)));
loader2 = *mask2vp++;
loader3 = _mm_and_si128(loader2, _mm_srli_epi64(loader, 1));
loader2 = _mm_and_si128(loader2, loader);
to_ct_abtmp = _mm_add_epi64(loader3, loader2);
to_ct2_c = _mm_add_epi64(to_ct2_c, _mm_andnot_si128(loader3, loader2));
to_ct2_ab = _mm_add_epi64(to_ct2_ab, _mm_add_epi64(_mm_and_si128(to_ct_abtmp, m2), _mm_and_si128(_mm_srli_epi64(to_ct_abtmp, 2), m2)));
loader = *geno_vvec++;
loader2 = *mask1vp++;
loader3 = _mm_and_si128(loader2, _mm_srli_epi64(loader, 1));
loader2 = _mm_and_si128(loader2, loader);
to_ct_abtmp = _mm_add_epi64(loader3, loader2);
to_ct1_c = _mm_add_epi64(to_ct1_c, _mm_andnot_si128(loader3, loader2));
to_ct1_ab = _mm_add_epi64(to_ct1_ab, _mm_add_epi64(_mm_and_si128(to_ct_abtmp, m2), _mm_and_si128(_mm_srli_epi64(to_ct_abtmp, 2), m2)));
loader2 = *mask2vp++;
loader3 = _mm_and_si128(loader2, _mm_srli_epi64(loader, 1));
loader2 = _mm_and_si128(loader2, loader);
to_ct_abtmp = _mm_add_epi64(loader3, loader2);
to_ct2_c = _mm_add_epi64(to_ct2_c, _mm_andnot_si128(loader3, loader2));
to_ct2_ab = _mm_add_epi64(to_ct2_ab, _mm_add_epi64(_mm_and_si128(to_ct_abtmp, m2), _mm_and_si128(_mm_srli_epi64(to_ct_abtmp, 2), m2)));
to_ct1_c = _mm_add_epi64(_mm_and_si128(to_ct1_c, m2), _mm_and_si128(_mm_srli_epi64(to_ct1_c, 2), m2));
to_ct2_c = _mm_add_epi64(_mm_and_si128(to_ct2_c, m2), _mm_and_si128(_mm_srli_epi64(to_ct2_c, 2), m2));
acc1_ab.vi = _mm_add_epi64(acc1_ab.vi, _mm_add_epi64(_mm_and_si128(to_ct1_ab, m4), _mm_and_si128(_mm_srli_epi64(to_ct1_ab, 4), m4)));
acc1_c.vi = _mm_add_epi64(acc1_c.vi, _mm_add_epi64(_mm_and_si128(to_ct1_c, m4), _mm_and_si128(_mm_srli_epi64(to_ct1_c, 4), m4)));
acc2_ab.vi = _mm_add_epi64(acc2_ab.vi, _mm_add_epi64(_mm_and_si128(to_ct2_ab, m4), _mm_and_si128(_mm_srli_epi64(to_ct2_ab, 4), m4)));
acc2_c.vi = _mm_add_epi64(acc2_c.vi, _mm_add_epi64(_mm_and_si128(to_ct2_c, m4), _mm_and_si128(_mm_srli_epi64(to_ct2_c, 4), m4)));
} while (geno_vvec < geno_vvec_end);
const __m128i m8 = {0x00ff00ff00ff00ffLLU, 0x00ff00ff00ff00ffLLU};
acc1_ab.vi = _mm_add_epi64(_mm_and_si128(acc1_ab.vi, m8), _mm_and_si128(_mm_srli_epi64(acc1_ab.vi, 8), m8));
acc1_c.vi = _mm_and_si128(_mm_add_epi64(acc1_c.vi, _mm_srli_epi64(acc1_c.vi, 8)), m8);
acc2_ab.vi = _mm_add_epi64(_mm_and_si128(acc2_ab.vi, m8), _mm_and_si128(_mm_srli_epi64(acc2_ab.vi, 8), m8));
acc2_c.vi = _mm_and_si128(_mm_add_epi64(acc2_c.vi, _mm_srli_epi64(acc2_c.vi, 8)), m8);
*ct1abp += ((acc1_ab.u8[0] + acc1_ab.u8[1]) * 0x1000100010001LLU) >> 48;
*ct1cp += ((acc1_c.u8[0] + acc1_c.u8[1]) * 0x1000100010001LLU) >> 48;
*ct2abp += ((acc2_ab.u8[0] + acc2_ab.u8[1]) * 0x1000100010001LLU) >> 48;
*ct2cp += ((acc2_c.u8[0] + acc2_c.u8[1]) * 0x1000100010001LLU) >> 48;
}
void count_3freq_1920b(const VECITYPE* geno_vvec, const VECITYPE* geno_vvec_end, const VECITYPE* __restrict maskvp, uint32_t* __restrict even_ctp, uint32_t* __restrict odd_ctp, uint32_t* __restrict homset_ctp) {
const __m128i m2 = {0x3333333333333333LLU, 0x3333333333333333LLU};
const __m128i m4 = {0x0f0f0f0f0f0f0f0fLLU, 0x0f0f0f0f0f0f0f0fLLU};
__m128i loader;
__m128i loader2;
__m128i loader3;
__m128i even1;
__m128i odd1;
__m128i homset1;
__m128i even2;
__m128i odd2;
__m128i homset2;
__univec acc_even;
__univec acc_odd;
__univec acc_homset;
acc_even.vi = _mm_setzero_si128();
acc_odd.vi = _mm_setzero_si128();
acc_homset.vi = _mm_setzero_si128();
do {
loader = *geno_vvec++;
loader2 = *maskvp++;
odd1 = _mm_and_si128(loader2, _mm_srli_epi64(loader, 1));
even1 = _mm_and_si128(loader2, loader);
homset1 = _mm_and_si128(odd1, loader);
loader = *geno_vvec++;
loader2 = *maskvp++;
loader3 = _mm_and_si128(loader2, _mm_srli_epi64(loader, 1));
even1 = _mm_add_epi64(even1, _mm_and_si128(loader2, loader));
odd1 = _mm_add_epi64(odd1, loader3);
homset1 = _mm_add_epi64(homset1, _mm_and_si128(loader3, loader));
loader = *geno_vvec++;
loader2 = *maskvp++;
loader3 = _mm_and_si128(loader2, _mm_srli_epi64(loader, 1));
even1 = _mm_add_epi64(even1, _mm_and_si128(loader2, loader));
odd1 = _mm_add_epi64(odd1, loader3);
homset1 = _mm_add_epi64(homset1, _mm_and_si128(loader3, loader));
even1 = _mm_add_epi64(_mm_and_si128(even1, m2), _mm_and_si128(_mm_srli_epi64(even1, 2), m2));
odd1 = _mm_add_epi64(_mm_and_si128(odd1, m2), _mm_and_si128(_mm_srli_epi64(odd1, 2), m2));
homset1 = _mm_add_epi64(_mm_and_si128(homset1, m2), _mm_and_si128(_mm_srli_epi64(homset1, 2), m2));
loader = *geno_vvec++;
loader2 = *maskvp++;
odd2 = _mm_and_si128(loader2, _mm_srli_epi64(loader, 1));
even2 = _mm_and_si128(loader2, loader);
homset2 = _mm_and_si128(odd2, loader);
loader = *geno_vvec++;
loader2 = *maskvp++;
loader3 = _mm_and_si128(loader2, _mm_srli_epi64(loader, 1));
even2 = _mm_add_epi64(even2, _mm_and_si128(loader2, loader));
odd2 = _mm_add_epi64(odd2, loader3);
homset2 = _mm_add_epi64(homset2, _mm_and_si128(loader3, loader));
loader = *geno_vvec++;
loader2 = *maskvp++;
loader3 = _mm_and_si128(loader2, _mm_srli_epi64(loader, 1));
even2 = _mm_add_epi64(even2, _mm_and_si128(loader2, loader));
odd2 = _mm_add_epi64(odd2, loader3);
homset2 = _mm_add_epi64(homset2, _mm_and_si128(loader3, loader));
even1 = _mm_add_epi64(even1, _mm_add_epi64(_mm_and_si128(even2, m2), _mm_and_si128(_mm_srli_epi64(even2, 2), m2)));
odd1 = _mm_add_epi64(odd1, _mm_add_epi64(_mm_and_si128(odd2, m2), _mm_and_si128(_mm_srli_epi64(odd2, 2), m2)));
homset1 = _mm_add_epi64(homset1, _mm_add_epi64(_mm_and_si128(homset2, m2), _mm_and_si128(_mm_srli_epi64(homset2, 2), m2)));
acc_even.vi = _mm_add_epi64(acc_even.vi, _mm_add_epi64(_mm_and_si128(even1, m4), _mm_and_si128(_mm_srli_epi64(even1, 4), m4)));
acc_odd.vi = _mm_add_epi64(acc_odd.vi, _mm_add_epi64(_mm_and_si128(odd1, m4), _mm_and_si128(_mm_srli_epi64(odd1, 4), m4)));
acc_homset.vi = _mm_add_epi64(acc_homset.vi, _mm_add_epi64(_mm_and_si128(homset1, m4), _mm_and_si128(_mm_srli_epi64(homset1, 4), m4)));
} while (geno_vvec < geno_vvec_end);
const __m128i m8 = {0x00ff00ff00ff00ffLLU, 0x00ff00ff00ff00ffLLU};
acc_even.vi = _mm_add_epi64(_mm_and_si128(acc_even.vi, m8), _mm_and_si128(_mm_srli_epi64(acc_even.vi, 8), m8));
acc_odd.vi = _mm_add_epi64(_mm_and_si128(acc_odd.vi, m8), _mm_and_si128(_mm_srli_epi64(acc_odd.vi, 8), m8));
acc_homset.vi = _mm_add_epi64(_mm_and_si128(acc_homset.vi, m8), _mm_and_si128(_mm_srli_epi64(acc_homset.vi, 8), m8));
*even_ctp += ((acc_even.u8[0] + acc_even.u8[1]) * 0x1000100010001LLU) >> 48;
*odd_ctp += ((acc_odd.u8[0] + acc_odd.u8[1]) * 0x1000100010001LLU) >> 48;
*homset_ctp += ((acc_homset.u8[0] + acc_homset.u8[1]) * 0x1000100010001LLU) >> 48;
}
#else
void count_2freq_dbl_24b(const uintptr_t* __restrict geno_vec, const uintptr_t* __restrict mask1p, const uintptr_t* __restrict mask2p, uint32_t* __restrict ct1abp, uint32_t* __restrict ct1cp, uint32_t* __restrict ct2abp, uint32_t* __restrict ct2cp) {
uintptr_t loader = *geno_vec++;
uintptr_t loader2 = *mask1p++;
uintptr_t loader3 = (loader >> 1) & loader2;
uintptr_t to_ct1_ab;
uintptr_t to_ct1_c;
uintptr_t to_ct2_ab;
uintptr_t to_ct2_c;
uintptr_t to_ct_abtmp;
uintptr_t partial1_ab;
uintptr_t partial1_c;
uintptr_t partial2_ab;
uintptr_t partial2_c;
loader2 &= loader;
to_ct1_ab = loader2 + loader3;
to_ct1_c = loader2 & (~loader3);
loader2 = *mask2p++;
loader3 = (loader >> 1) & loader2;
loader2 &= loader;
to_ct2_ab = loader2 + loader3;
to_ct2_c = loader2 & (~loader3);
to_ct1_ab = (to_ct1_ab & 0x33333333) + ((to_ct1_ab >> 2) & 0x33333333);
to_ct2_ab = (to_ct2_ab & 0x33333333) + ((to_ct2_ab >> 2) & 0x33333333);
loader = *geno_vec++;
loader2 = *mask1p++;
loader3 = (loader >> 1) & loader2;
loader2 &= loader;
to_ct_abtmp = loader2 + loader3;
to_ct1_c += loader2 & (~loader3);
to_ct1_ab += (to_ct_abtmp & 0x33333333) + ((to_ct_abtmp >> 2) & 0x33333333);
loader2 = *mask2p++;
loader3 = (loader >> 1) & loader2;
loader2 &= loader;
to_ct_abtmp = loader2 + loader3;
to_ct2_c += loader2 & (~loader3);
to_ct2_ab += (to_ct_abtmp & 0x33333333) + ((to_ct_abtmp >> 2) & 0x33333333);
loader = *geno_vec++;
loader2 = *mask1p++;
loader3 = (loader >> 1) & loader2;
loader2 &= loader;
to_ct_abtmp = loader2 + loader3;
to_ct1_c += loader2 & (~loader3);
to_ct1_ab += (to_ct_abtmp & 0x33333333) + ((to_ct_abtmp >> 2) & 0x33333333);
loader2 = *mask2p++;
loader3 = (loader >> 1) & loader2;
loader2 &= loader;
to_ct_abtmp = loader2 + loader3;
to_ct2_c += loader2 & (~loader3);
to_ct2_ab += (to_ct_abtmp & 0x33333333) + ((to_ct_abtmp >> 2) & 0x33333333);
partial1_ab = (to_ct1_ab & 0x0f0f0f0f) + ((to_ct1_ab >> 4) & 0x0f0f0f0f);
partial1_c = (to_ct1_c & 0x33333333) + ((to_ct1_c >> 2) & 0x33333333);
partial2_ab = (to_ct2_ab & 0x0f0f0f0f) + ((to_ct2_ab >> 4) & 0x0f0f0f0f);
partial2_c = (to_ct2_c & 0x33333333) + ((to_ct2_c >> 2) & 0x33333333);
loader = *geno_vec++;
loader2 = *mask1p++;
loader3 = (loader >> 1) & loader2;
loader2 &= loader;
to_ct1_ab = loader2 + loader3;
to_ct1_c = loader2 & (~loader3);
loader2 = *mask2p++;
loader3 = (loader >> 1) & loader2;
loader2 &= loader;
to_ct2_ab = loader2 + loader3;
to_ct2_c = loader2 & (~loader3);
to_ct1_ab = (to_ct1_ab & 0x33333333) + ((to_ct1_ab >> 2) & 0x33333333);
to_ct2_ab = (to_ct2_ab & 0x33333333) + ((to_ct2_ab >> 2) & 0x33333333);
loader = *geno_vec++;
loader2 = *mask1p++;
loader3 = (loader >> 1) & loader2;
loader2 &= loader;
to_ct_abtmp = loader2 + loader3;
to_ct1_c += loader2 & (~loader3);
to_ct1_ab += (to_ct_abtmp & 0x33333333) + ((to_ct_abtmp >> 2) & 0x33333333);
loader2 = *mask2p++;
loader3 = (loader >> 1) & loader2;
loader2 &= loader;
to_ct_abtmp = loader2 + loader3;
to_ct2_c += loader2 & (~loader3);
to_ct2_ab += (to_ct_abtmp & 0x33333333) + ((to_ct_abtmp >> 2) & 0x33333333);
loader = *geno_vec++;
loader2 = *mask1p++;
loader3 = (loader >> 1) & loader2;
loader2 &= loader;
to_ct_abtmp = loader2 + loader3;
to_ct1_c += loader2 & (~loader3);
to_ct1_ab += (to_ct_abtmp & 0x33333333) + ((to_ct_abtmp >> 2) & 0x33333333);
loader2 = *mask2p++;
loader3 = (loader >> 1) & loader2;
loader2 &= loader;
to_ct_abtmp = loader2 + loader3;
to_ct2_c += loader2 & (~loader3);
to_ct2_ab += (to_ct_abtmp & 0x33333333) + ((to_ct_abtmp >> 2) & 0x33333333);
partial1_ab += (to_ct1_ab & 0x0f0f0f0f) + ((to_ct1_ab >> 4) & 0x0f0f0f0f);
partial1_c += (to_ct1_c & 0x33333333) + ((to_ct1_c >> 2) & 0x33333333);
partial2_ab += (to_ct2_ab & 0x0f0f0f0f) + ((to_ct2_ab >> 4) & 0x0f0f0f0f);
partial2_c += (to_ct2_c & 0x33333333) + ((to_ct2_c >> 2) & 0x33333333);
partial1_c = (partial1_c & 0x0f0f0f0f) + ((partial1_c >> 4) & 0x0f0f0f0f);
partial2_c = (partial2_c & 0x0f0f0f0f) + ((partial2_c >> 4) & 0x0f0f0f0f);
*ct1abp += (partial1_ab * 0x01010101) >> 24;
*ct1cp += (partial1_c * 0x01010101) >> 24;
*ct2abp += (partial2_ab * 0x01010101) >> 24;
*ct2cp += (partial2_c * 0x01010101) >> 24;
}
void count_3freq_48b(const uintptr_t* __restrict geno_vec, const uintptr_t* __restrict maskp, uint32_t* __restrict ctap, uint32_t* __restrict ctbp, uint32_t* __restrict ctcp) {
uintptr_t loader = *geno_vec++;
uintptr_t loader2 = *maskp++;
uint32_t to_ct_a1 = loader & loader2;
uint32_t to_ct_b1 = (loader >> 1) & loader2;
uint32_t to_ct_c1 = loader & to_ct_b1;
uintptr_t loader3;
uint32_t to_ct_a2;
uint32_t to_ct_b2;
uint32_t to_ct_c2;
uintptr_t partial_a;
uintptr_t partial_b;
uintptr_t partial_c;
loader = *geno_vec++;
loader2 = *maskp++;
loader3 = (loader >> 1) & loader2;
to_ct_a1 += loader & loader2;
to_ct_b1 += loader3;
to_ct_c1 += loader & loader3;
loader = *geno_vec++;
loader2 = *maskp++;
loader3 = (loader >> 1) & loader2;
to_ct_a1 += loader & loader2;
to_ct_b1 += loader3;
to_ct_c1 += loader & loader3;
loader = *geno_vec++;
loader2 = *maskp++;
to_ct_a2 = loader & loader2;
to_ct_b2 = (loader >> 1) & loader2;
to_ct_c2 = loader & to_ct_b2;
loader = *geno_vec++;
loader2 = *maskp++;
loader3 = (loader >> 1) & loader2;
to_ct_a2 += loader & loader2;
to_ct_b2 += loader3;
to_ct_c2 += loader & loader3;
loader = *geno_vec++;
loader2 = *maskp++;
loader3 = (loader >> 1) & loader2;
to_ct_a2 += loader & loader2;
to_ct_b2 += loader3;
to_ct_c2 += loader & loader3;
to_ct_a1 = (to_ct_a1 & 0x33333333) + ((to_ct_a1 >> 2) & 0x33333333);
to_ct_a1 += (to_ct_a2 & 0x33333333) + ((to_ct_a2 >> 2) & 0x33333333);
partial_a = (to_ct_a1 & 0x0f0f0f0f) + ((to_ct_a1 >> 4) & 0x0f0f0f0f);
to_ct_b1 = (to_ct_b1 & 0x33333333) + ((to_ct_b1 >> 2) & 0x33333333);
to_ct_b1 += (to_ct_b2 & 0x33333333) + ((to_ct_b2 >> 2) & 0x33333333);
partial_b = (to_ct_b1 & 0x0f0f0f0f) + ((to_ct_b1 >> 4) & 0x0f0f0f0f);
to_ct_c1 = (to_ct_c1 & 0x33333333) + ((to_ct_c1 >> 2) & 0x33333333);
to_ct_c1 += (to_ct_c2 & 0x33333333) + ((to_ct_c2 >> 2) & 0x33333333);
partial_c = (to_ct_c1 & 0x0f0f0f0f) + ((to_ct_c1 >> 4) & 0x0f0f0f0f);
loader = *geno_vec++;
loader2 = *maskp++;
to_ct_a1 = loader & loader2;
to_ct_b1 = (loader >> 1) & loader2;
to_ct_c1 = loader & to_ct_b1;
loader = *geno_vec++;
loader2 = *maskp++;
loader3 = (loader >> 1) & loader2;
to_ct_a1 += loader & loader2;
to_ct_b1 += loader3;
to_ct_c1 += loader & loader3;
loader = *geno_vec++;
loader2 = *maskp++;
loader3 = (loader >> 1) & loader2;
to_ct_a1 += loader & loader2;
to_ct_b1 += loader3;
to_ct_c1 += loader & loader3;
loader = *geno_vec++;
loader2 = *maskp++;
to_ct_a2 = loader & loader2;
to_ct_b2 = (loader >> 1) & loader2;
to_ct_c2 = loader & to_ct_b2;
loader = *geno_vec++;
loader2 = *maskp++;
loader3 = (loader >> 1) & loader2;
to_ct_a2 += loader & loader2;
to_ct_b2 += loader3;
to_ct_c2 += loader & loader3;
loader = *geno_vec;
loader2 = *maskp;
loader3 = (loader >> 1) & loader2;
to_ct_a2 += loader & loader2;
to_ct_b2 += loader3;
to_ct_c2 += loader & loader3;
to_ct_a1 = (to_ct_a1 & 0x33333333) + ((to_ct_a1 >> 2) & 0x33333333);
to_ct_a1 += (to_ct_a2 & 0x33333333) + ((to_ct_a2 >> 2) & 0x33333333);
partial_a += (to_ct_a1 & 0x0f0f0f0f) + ((to_ct_a1 >> 4) & 0x0f0f0f0f);
to_ct_b1 = (to_ct_b1 & 0x33333333) + ((to_ct_b1 >> 2) & 0x33333333);
to_ct_b1 += (to_ct_b2 & 0x33333333) + ((to_ct_b2 >> 2) & 0x33333333);
partial_b += (to_ct_b1 & 0x0f0f0f0f) + ((to_ct_b1 >> 4) & 0x0f0f0f0f);
to_ct_c1 = (to_ct_c1 & 0x33333333) + ((to_ct_c1 >> 2) & 0x33333333);
to_ct_c1 += (to_ct_c2 & 0x33333333) + ((to_ct_c2 >> 2) & 0x33333333);
partial_c += (to_ct_c1 & 0x0f0f0f0f) + ((to_ct_c1 >> 4) & 0x0f0f0f0f);
*ctap += (partial_a * 0x01010101) >> 24;
*ctbp += (partial_b * 0x01010101) >> 24;
*ctcp += (partial_c * 0x01010101) >> 24;
}
#endif
#ifdef __LP64__
void count_set_freq_60v(const __m128i* vptr, const __m128i* vend, const __m128i* __restrict include_vec, uint32_t* __restrict set_ctp, uint32_t* __restrict missing_ctp) {
const __m128i m2 = {0x3333333333333333LLU, 0x3333333333333333LLU};
const __m128i m4 = {0x0f0f0f0f0f0f0f0fLLU, 0x0f0f0f0f0f0f0f0fLLU};
const __m128i m8 = {0x00ff00ff00ff00ffLLU, 0x00ff00ff00ff00ffLLU};
__m128i loader;
__m128i loader2;
__m128i loader3;
__m128i odds;
__m128i evens;
__m128i missings;
__univec acc;
__univec accm;
acc.vi = _mm_setzero_si128();
accm.vi = _mm_setzero_si128();
do {
loader = *vptr++;
loader2 = _mm_srli_epi64(loader, 1);
loader3 = *include_vec++;
odds = _mm_and_si128(loader2, loader3);
evens = _mm_and_si128(odds, loader);
missings = _mm_and_si128(loader, _mm_andnot_si128(loader2, loader3));
loader = *vptr++;
loader2 = _mm_srli_epi64(loader, 1);
loader3 = *include_vec++;
odds = _mm_add_epi64(odds, _mm_and_si128(loader2, loader3));
loader3 = _mm_and_si128(loader, loader3);
evens = _mm_add_epi64(evens, _mm_and_si128(loader2, loader3));
missings = _mm_add_epi64(missings, _mm_andnot_si128(loader2, loader3));
loader = *vptr++;
loader2 = _mm_srli_epi64(loader, 1);
loader3 = *include_vec++;
odds = _mm_add_epi64(odds, _mm_and_si128(loader2, loader3));
loader3 = _mm_and_si128(loader, loader3);
evens = _mm_add_epi64(evens, _mm_and_si128(loader2, loader3));
missings = _mm_add_epi64(missings, _mm_andnot_si128(loader2, loader3));
odds = _mm_add_epi64(_mm_and_si128(odds, m2), _mm_and_si128(_mm_srli_epi64(odds, 2), m2));
missings = _mm_add_epi64(_mm_and_si128(missings, m2), _mm_and_si128(_mm_srli_epi64(missings, 2), m2));
odds = _mm_add_epi64(odds, _mm_add_epi64(_mm_and_si128(evens, m2), _mm_and_si128(_mm_srli_epi64(evens, 2), m2)));
// each 4-bit value here <= 6, so safe to add before m4 mask
accm.vi = _mm_add_epi64(accm.vi, _mm_and_si128(_mm_add_epi64(missings, _mm_srli_epi64(missings, 4)), m4));
acc.vi = _mm_add_epi64(acc.vi, _mm_add_epi64(_mm_and_si128(odds, m4), _mm_and_si128(_mm_srli_epi64(odds, 4), m4)));
} while (vptr < vend);
// and each 8-bit value here <= 120
accm.vi = _mm_and_si128(_mm_add_epi64(accm.vi, _mm_srli_epi64(accm.vi, 8)), m8);
acc.vi = _mm_add_epi64(_mm_and_si128(acc.vi, m8), _mm_and_si128(_mm_srli_epi64(acc.vi, 8), m8));
*set_ctp += ((acc.u8[0] + acc.u8[1]) * 0x1000100010001LLU) >> 48;
*missing_ctp += ((accm.u8[0] + accm.u8[1]) * 0x1000100010001LLU) >> 48;
}
void count_set_freq_hap_120v(const __m128i* vptr, const __m128i* vend, const __m128i* __restrict include_vec, uint32_t* __restrict set_ctp, uint32_t* __restrict missing_ctp) {
const __m128i m2 = {0x3333333333333333LLU, 0x3333333333333333LLU};
const __m128i m4 = {0x0f0f0f0f0f0f0f0fLLU, 0x0f0f0f0f0f0f0f0fLLU};
const __m128i m8 = {0x00ff00ff00ff00ffLLU, 0x00ff00ff00ff00ffLLU};
__univec acc;
__univec accm;
__m128i loader;
__m128i loader2;
__m128i loader3;
__m128i partial;
__m128i partialm;
__m128i partial2;
__m128i partial2m;
acc.vi = _mm_setzero_si128();
accm.vi = _mm_setzero_si128();
do {
loader = *vptr++;
loader2 = _mm_srli_epi64(loader, 1);
loader3 = *include_vec++;
partial = _mm_and_si128(loader3, _mm_and_si128(loader, loader2));
partialm = _mm_and_si128(loader3, _mm_xor_si128(loader, loader2));
loader = *vptr++;
loader2 = _mm_srli_epi64(loader, 1);
loader3 = *include_vec++;
partial = _mm_add_epi64(partial, _mm_and_si128(loader3, _mm_and_si128(loader, loader2)));
partialm = _mm_add_epi64(partialm, _mm_and_si128(loader3, _mm_and_si128(loader, loader2)));
loader = *vptr++;
loader2 = _mm_srli_epi64(loader, 1);
loader3 = *include_vec++;
partial = _mm_add_epi64(partial, _mm_and_si128(loader3, _mm_and_si128(loader, loader2)));
partialm = _mm_add_epi64(partialm, _mm_and_si128(loader3, _mm_and_si128(loader, loader2)));
partial2 = _mm_add_epi64(_mm_and_si128(partial, m2), _mm_and_si128(_mm_srli_epi64(partial, 2), m2));
partial2m = _mm_add_epi64(_mm_and_si128(partialm, m2), _mm_and_si128(_mm_srli_epi64(partialm, 2), m2));
loader = *vptr++;
loader2 = _mm_srli_epi64(loader, 1);
loader3 = *include_vec++;
partial = _mm_and_si128(loader3, _mm_and_si128(loader, loader2));
partialm = _mm_and_si128(loader3, _mm_xor_si128(loader, loader2));
loader = *vptr++;
loader2 = _mm_srli_epi64(loader, 1);
loader3 = *include_vec++;
partial = _mm_add_epi64(partial, _mm_and_si128(loader3, _mm_and_si128(loader, loader2)));
partialm = _mm_add_epi64(partialm, _mm_and_si128(loader3, _mm_and_si128(loader, loader2)));
loader = *vptr++;
loader2 = _mm_srli_epi64(loader, 1);
loader3 = *include_vec++;
partial = _mm_add_epi64(partial, _mm_and_si128(loader3, _mm_and_si128(loader, loader2)));
partialm = _mm_add_epi64(partialm, _mm_and_si128(loader3, _mm_and_si128(loader, loader2)));
partial2 = _mm_add_epi64(partial2, _mm_add_epi64(_mm_and_si128(partial, m2), _mm_and_si128(_mm_srli_epi64(partial, 2), m2)));
partial2m = _mm_add_epi64(partial2m, _mm_add_epi64(_mm_and_si128(partialm, m2), _mm_and_si128(_mm_srli_epi64(partialm, 2), m2)));
acc.vi = _mm_add_epi64(acc.vi, _mm_add_epi64(_mm_and_si128(partial2, m4), _mm_and_si128(_mm_srli_epi64(partial2, 4), m4)));
accm.vi = _mm_add_epi64(accm.vi, _mm_add_epi64(_mm_and_si128(partial2m, m4), _mm_and_si128(_mm_srli_epi64(partial2m, 4), m4)));
} while (vptr < vend);
acc.vi = _mm_add_epi64(_mm_and_si128(acc.vi, m8), _mm_and_si128(_mm_srli_epi64(acc.vi, 8), m8));
accm.vi = _mm_add_epi64(_mm_and_si128(accm.vi, m8), _mm_and_si128(_mm_srli_epi64(accm.vi, 8), m8));
*set_ctp += ((acc.u8[0] + acc.u8[1]) * 0x1000100010001LLU) >> 48;
*missing_ctp += ((accm.u8[0] + accm.u8[1]) * 0x1000100010001LLU) >> 48;
}
void count_set_freq_x_60v(const __m128i* vptr, const __m128i* vend, const __m128i* __restrict include_vec, const __m128i* __restrict male_vec, uint32_t* __restrict set_ctp, uint32_t* __restrict missing_ctp) {
const __m128i m2 = {0x3333333333333333LLU, 0x3333333333333333LLU};
const __m128i m4 = {0x0f0f0f0f0f0f0f0fLLU, 0x0f0f0f0f0f0f0f0fLLU};
const __m128i m8 = {0x00ff00ff00ff00ffLLU, 0x00ff00ff00ff00ffLLU};
__m128i loader;
__m128i loader2;
__m128i loader3;
__m128i loader4;
__m128i set_odds;
__m128i set_evens;
__m128i missings_nm;
__m128i missings_m;
__m128i males;
__univec acc;
__univec accm;
acc.vi = _mm_setzero_si128();
accm.vi = _mm_setzero_si128();
do {
loader = *vptr++;
loader2 = _mm_srli_epi64(loader, 1);
loader3 = *include_vec++;
loader4 = _mm_andnot_si128(*male_vec, loader3);
set_evens = _mm_and_si128(loader, loader4); // subtract missings_nm later
set_odds = _mm_and_si128(loader2, loader4);
missings_nm = _mm_andnot_si128(loader2, set_evens);
males = _mm_and_si128(loader3, *male_vec++);
set_evens = _mm_or_si128(set_evens, _mm_and_si128(_mm_and_si128(loader, loader2), males));
missings_m = _mm_and_si128(_mm_xor_si128(loader, loader2), males);
loader = *vptr++;
loader2 = _mm_srli_epi64(loader, 1);
loader3 = *include_vec++;
loader4 = _mm_andnot_si128(*male_vec, loader3);
set_odds = _mm_add_epi64(set_odds, _mm_and_si128(loader2, loader4));
loader4 = _mm_and_si128(loader, loader4);
set_evens = _mm_add_epi64(set_evens, loader4);
missings_nm = _mm_add_epi64(missings_nm, _mm_andnot_si128(loader2, loader4));
loader4 = _mm_and_si128(loader3, *male_vec++);
set_evens = _mm_add_epi64(set_evens, _mm_and_si128(_mm_and_si128(loader, loader2), loader4));
missings_m = _mm_add_epi64(missings_m, _mm_and_si128(_mm_xor_si128(loader, loader2), loader4));
males = _mm_add_epi64(males, loader4);
loader = *vptr++;
loader2 = _mm_srli_epi64(loader, 1);
loader3 = *include_vec++;
loader4 = _mm_andnot_si128(*male_vec, loader3);
set_odds = _mm_add_epi64(set_odds, _mm_and_si128(loader2, loader4));
loader4 = _mm_and_si128(loader, loader4);
set_evens = _mm_add_epi64(set_evens, loader4);
missings_nm = _mm_add_epi64(missings_nm, _mm_andnot_si128(loader2, loader4));
loader4 = _mm_and_si128(loader3, *male_vec++);
set_evens = _mm_add_epi64(set_evens, _mm_and_si128(_mm_and_si128(loader, loader2), loader4));
missings_m = _mm_add_epi64(missings_m, _mm_and_si128(_mm_xor_si128(loader, loader2), loader4));
males = _mm_add_epi64(males, loader4);
set_evens = _mm_sub_epi64(set_evens, missings_nm);
missings_nm = _mm_slli_epi64(_mm_add_epi64(_mm_and_si128(missings_nm, m2), _mm_and_si128(_mm_srli_epi64(missings_nm, 2), m2)), 1);
set_odds = _mm_add_epi64(_mm_and_si128(set_odds, m2), _mm_and_si128(_mm_srli_epi64(set_odds, 2), m2));
missings_nm = _mm_add_epi64(missings_nm, _mm_add_epi64(_mm_and_si128(missings_m, m2), _mm_and_si128(_mm_srli_epi64(missings_m, 2), m2)));
set_odds = _mm_add_epi64(set_odds, _mm_add_epi64(_mm_and_si128(set_evens, m2), _mm_and_si128(_mm_srli_epi64(set_evens, 2), m2)));
missings_nm = _mm_add_epi64(missings_nm, _mm_add_epi64(_mm_and_si128(males, m2), _mm_and_si128(_mm_srli_epi64(males, 2), m2)));
acc.vi = _mm_add_epi64(acc.vi, _mm_add_epi64(_mm_and_si128(set_odds, m4), _mm_and_si128(_mm_srli_epi64(set_odds, 4), m4)));
accm.vi = _mm_add_epi64(accm.vi, _mm_add_epi64(_mm_and_si128(missings_nm, m4), _mm_and_si128(_mm_srli_epi64(missings_nm, 4), m4)));
} while (vptr < vend);
acc.vi = _mm_add_epi64(_mm_and_si128(acc.vi, m8), _mm_and_si128(_mm_srli_epi64(acc.vi, 8), m8));
accm.vi = _mm_add_epi64(_mm_and_si128(accm.vi, m8), _mm_and_si128(_mm_srli_epi64(accm.vi, 8), m8));
*set_ctp += ((acc.u8[0] + acc.u8[1]) * 0x1000100010001LLU) >> 48;
*missing_ctp += ((accm.u8[0] + accm.u8[1]) * 0x1000100010001LLU) >> 48;
}
void count_set_freq_y_120v(const __m128i* vptr, const __m128i* vend, const __m128i* __restrict include_vec, const __m128i* __restrict nonmale_vec, uint32_t* __restrict set_ctp, uint32_t* __restrict missing_ctp) {
const __m128i m2 = {0x3333333333333333LLU, 0x3333333333333333LLU};
const __m128i m4 = {0x0f0f0f0f0f0f0f0fLLU, 0x0f0f0f0f0f0f0f0fLLU};
const __m128i m8 = {0x00ff00ff00ff00ffLLU, 0x00ff00ff00ff00ffLLU};
__m128i loader;
__m128i loader2;
__m128i loader3;
__m128i loader4;
__m128i sets1;
__m128i missings1;
__m128i sets2;
__m128i missings2;
__univec acc;
__univec accm;
acc.vi = _mm_setzero_si128();
accm.vi = _mm_setzero_si128();
do {
loader = *vptr++;
loader3 = *include_vec++;
loader2 = _mm_srli_epi64(loader, 1);
loader4 = *nonmale_vec++;
sets1 = _mm_and_si128(_mm_andnot_si128(loader4, loader3), _mm_and_si128(loader, loader2));
missings1 = _mm_and_si128(loader3, _mm_or_si128(loader4, _mm_xor_si128(loader, loader2)));
loader = *vptr++;
loader3 = *include_vec++;
loader2 = _mm_srli_epi64(loader, 1);
loader4 = *nonmale_vec++;
sets1 = _mm_add_epi64(sets1, _mm_and_si128(_mm_andnot_si128(loader4, loader3), _mm_and_si128(loader, loader2)));
missings1 = _mm_add_epi64(missings1, _mm_and_si128(loader3, _mm_or_si128(loader4, _mm_xor_si128(loader, loader2))));
loader = *vptr++;
loader3 = *include_vec++;
loader2 = _mm_srli_epi64(loader, 1);
loader4 = *nonmale_vec++;
sets1 = _mm_add_epi64(sets1, _mm_and_si128(_mm_andnot_si128(loader4, loader3), _mm_and_si128(loader, loader2)));
missings1 = _mm_add_epi64(missings1, _mm_and_si128(loader3, _mm_or_si128(loader4, _mm_xor_si128(loader, loader2))));
sets1 = _mm_add_epi64(_mm_and_si128(sets1, m2), _mm_and_si128(_mm_srli_epi64(sets1, 2), m2));
missings1 = _mm_add_epi64(_mm_and_si128(missings1, m2), _mm_and_si128(_mm_srli_epi64(missings1, 2), m2));
loader = *vptr++;
loader3 = *include_vec++;
loader2 = _mm_srli_epi64(loader, 1);
loader4 = *nonmale_vec++;
sets2 = _mm_and_si128(_mm_andnot_si128(loader4, loader3), _mm_and_si128(loader, loader2));
missings2 = _mm_and_si128(loader3, _mm_or_si128(loader4, _mm_xor_si128(loader, loader2)));
loader = *vptr++;
loader3 = *include_vec++;
loader2 = _mm_srli_epi64(loader, 1);
loader4 = *nonmale_vec++;
sets2 = _mm_add_epi64(sets2, _mm_and_si128(_mm_andnot_si128(loader4, loader3), _mm_and_si128(loader, loader2)));
missings2 = _mm_add_epi64(missings2, _mm_and_si128(loader3, _mm_or_si128(loader4, _mm_xor_si128(loader, loader2))));
loader = *vptr++;
loader3 = *include_vec++;
loader2 = _mm_srli_epi64(loader, 1);
loader4 = *nonmale_vec++;
sets2 = _mm_add_epi64(sets2, _mm_and_si128(_mm_andnot_si128(loader4, loader3), _mm_and_si128(loader, loader2)));
missings2 = _mm_add_epi64(missings2, _mm_and_si128(loader3, _mm_or_si128(loader4, _mm_xor_si128(loader, loader2))));
sets1 = _mm_add_epi64(sets1, _mm_add_epi64(_mm_and_si128(sets2, m2), _mm_and_si128(_mm_srli_epi64(sets2, 2), m2)));
missings1 = _mm_add_epi64(missings1, _mm_add_epi64(_mm_and_si128(missings2, m2), _mm_and_si128(_mm_srli_epi64(missings2, 2), m2)));
acc.vi = _mm_add_epi64(acc.vi, _mm_add_epi64(_mm_and_si128(sets1, m4), _mm_and_si128(_mm_srli_epi64(sets1, 4), m4)));
accm.vi = _mm_add_epi64(accm.vi, _mm_add_epi64(_mm_and_si128(missings1, m4), _mm_and_si128(_mm_srli_epi64(missings1, 4), m4)));
} while (vptr < vend);
acc.vi = _mm_add_epi64(_mm_and_si128(acc.vi, m8), _mm_and_si128(_mm_srli_epi64(acc.vi, 8), m8));
accm.vi = _mm_add_epi64(_mm_and_si128(accm.vi, m8), _mm_and_si128(_mm_srli_epi64(accm.vi, 8), m8));
*set_ctp += ((acc.u8[0] + acc.u8[1]) * 0x1000100010001LLU) >> 48;
*missing_ctp += ((accm.u8[0] + accm.u8[1]) * 0x1000100010001LLU) >> 48;
}
uintptr_t count_01_vecs(const __m128i* vptr, uintptr_t vct) {
// counts number of aligned 01s (i.e. PLINK missing genotypes) in
// [vptr, vend). Assumes number of words in interval is a multiple of 12.
const __m128i m1 = {FIVEMASK, FIVEMASK};
const __m128i m2 = {0x3333333333333333LLU, 0x3333333333333333LLU};
const __m128i m4 = {0x0f0f0f0f0f0f0f0fLLU, 0x0f0f0f0f0f0f0f0fLLU};
const __m128i m8 = {0x00ff00ff00ff00ffLLU, 0x00ff00ff00ff00ffLLU};
uintptr_t tot = 0;
const __m128i* vend;
__m128i loader1;
__m128i loader2;
__m128i count1;
__m128i count2;
__univec acc;
while (vct >= 60) {
vct -= 60;
vend = &(vptr[60]);
count_01_vecs_main_loop:
acc.vi = _mm_setzero_si128();
do {
loader1 = *vptr++;
loader2 = *vptr++;
count1 = _mm_and_si128(_mm_andnot_si128(_mm_srli_epi64(loader1, 1), loader1), m1);
count2 = _mm_and_si128(_mm_andnot_si128(_mm_srli_epi64(loader2, 1), loader2), m1);
loader1 = *vptr++;
loader2 = *vptr++;
count1 = _mm_add_epi64(count1, _mm_and_si128(_mm_andnot_si128(_mm_srli_epi64(loader1, 1), loader1), m1));
count2 = _mm_add_epi64(count2, _mm_and_si128(_mm_andnot_si128(_mm_srli_epi64(loader2, 1), loader2), m1));
loader1 = *vptr++;
loader2 = *vptr++;
count1 = _mm_add_epi64(count1, _mm_and_si128(_mm_andnot_si128(_mm_srli_epi64(loader1, 1), loader1), m1));
count2 = _mm_add_epi64(count2, _mm_and_si128(_mm_andnot_si128(_mm_srli_epi64(loader2, 1), loader2), m1));
count1 = _mm_add_epi64(_mm_and_si128(count1, m2), _mm_and_si128(_mm_srli_epi64(count1, 2), m2));
count1 = _mm_add_epi64(count1, _mm_add_epi64(_mm_and_si128(count2, m2), _mm_and_si128(_mm_srli_epi64(count2, 2), m2)));
acc.vi = _mm_add_epi64(acc.vi, _mm_add_epi64(_mm_and_si128(count1, m4), _mm_and_si128(_mm_srli_epi64(count1, 4), m4)));
} while (vptr < vend);
acc.vi = _mm_add_epi64(_mm_and_si128(acc.vi, m8), _mm_and_si128(_mm_srli_epi64(acc.vi, 8), m8));
tot += ((acc.u8[0] + acc.u8[1]) * 0x1000100010001LLU) >> 48;
}
if (vct) {
vend = &(vptr[vct]);
vct = 0;
goto count_01_vecs_main_loop;
}
return tot;
}
#else
void count_set_freq_6(const uintptr_t* __restrict lptr, const uintptr_t* __restrict include_vec, uint32_t* __restrict set_ctp, uint32_t* __restrict missing_ctp) {
uintptr_t loader = *lptr++;
uintptr_t loader2 = loader >> 1;
uintptr_t loader3 = *include_vec++;
uintptr_t odds = loader2 & loader3;
uintptr_t evens = odds & loader;
uintptr_t missings = (~loader2) & loader3 & loader;
uintptr_t acc;
uintptr_t accm;
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
odds += loader2 & loader3;
loader3 &= loader;
evens += loader2 & loader3;
missings += (~loader2) & loader3;
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
odds += loader2 & loader3;
loader3 &= loader;
evens += loader2 & loader3;
missings += (~loader2) & loader3;
odds = (odds & 0x33333333) + ((odds >> 2) & 0x33333333);
odds += (evens & 0x33333333) + ((evens >> 2) & 0x33333333);
accm = (missings & 0x33333333) + ((missings >> 2) & 0x33333333);
acc = (odds & 0x0f0f0f0f) + ((odds >> 4) & 0x0f0f0f0f);
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
odds = loader2 & loader3;
evens = odds & loader;
missings = (~loader2) & loader3 & loader;
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
odds += loader2 & loader3;
loader3 &= loader;
evens += loader2 & loader3;
missings += (~loader2) & loader3;
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
odds += loader2 & loader3;
loader3 &= loader;
evens += loader2 & loader3;
missings += (~loader2) & loader3;
odds = (odds & 0x33333333) + ((odds >> 2) & 0x33333333);
accm += (missings & 0x33333333) + ((missings >> 2) & 0x33333333);
odds += (evens & 0x33333333) + ((evens >> 2) & 0x33333333);
accm = (accm & 0x0f0f0f0f) + ((accm >> 4) & 0x0f0f0f0f);
acc += (odds & 0x0f0f0f0f) + ((odds >> 4) & 0x0f0f0f0f);
*set_ctp += (acc * 0x01010101) >> 24;
*missing_ctp += (accm * 0x01010101) >> 24;
}
void count_set_freq_hap_12(const uintptr_t* __restrict lptr, const uintptr_t* __restrict include_vec, uint32_t* __restrict set_ctp, uint32_t* __restrict missing_ctp) {
uintptr_t loader = *lptr++;
uintptr_t loader2 = loader >> 1;
uintptr_t loader3 = *include_vec++;
uintptr_t partial = loader & loader2 & loader3;
uintptr_t partialm = (loader ^ loader2) & loader3;
uintptr_t partial2;
uintptr_t partial2m;
uintptr_t acc;
uintptr_t accm;
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
partial += loader & loader2 & loader3;
partialm += (loader ^ loader2) & loader3;
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
partial += loader & loader2 & loader3;
partialm += (loader ^ loader2) & loader3;
partial2 = (partial & 0x33333333) + ((partial >> 2) & 0x33333333);
partial2m = (partialm & 0x33333333) + ((partialm >> 2) & 0x33333333);
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
partial = loader & loader2 & loader3;
partialm = (loader ^ loader2) & loader3;
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
partial += loader & loader2 & loader3;
partialm += (loader ^ loader2) & loader3;
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
partial += loader & loader2 & loader3;
partialm += (loader ^ loader2) & loader3;
partial2 += (partial & 0x33333333) + ((partial >> 2) & 0x33333333);
partial2m += (partialm & 0x33333333) + ((partialm >> 2) & 0x33333333);
acc = (partial2 & 0x0f0f0f0f) + ((partial2 >> 4) & 0x0f0f0f0f);
accm = (partial2m & 0x0f0f0f0f) + ((partial2m >> 4) & 0x0f0f0f0f);
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
partial = loader & loader2 & loader3;
partialm = (loader ^ loader2) & loader3;
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
partial += loader & loader2 & loader3;
partialm += (loader ^ loader2) & loader3;
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
partial += loader & loader2 & loader3;
partialm += (loader ^ loader2) & loader3;
partial2 = (partial & 0x33333333) + ((partial >> 2) & 0x33333333);
partial2m = (partialm & 0x33333333) + ((partialm >> 2) & 0x33333333);
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
partial = loader & loader2 & loader3;
partialm = (loader ^ loader2) & loader3;
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
partial += loader & loader2 & loader3;
partialm += (loader ^ loader2) & loader3;
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
partial += loader & loader2 & loader3;
partialm += (loader ^ loader2) & loader3;
partial2 += (partial & 0x33333333) + ((partial >> 2) & 0x33333333);
partial2m += (partialm & 0x33333333) + ((partialm >> 2) & 0x33333333);
acc += (partial2 & 0x0f0f0f0f) + ((partial2 >> 4) & 0x0f0f0f0f);
accm += (partial2m & 0x0f0f0f0f) + ((partial2m >> 4) & 0x0f0f0f0f);
*set_ctp += (acc * 0x01010101) >> 24;
*missing_ctp += (accm * 0x01010101) >> 24;
}
void count_set_freq_x_6(const uintptr_t* __restrict lptr, const uintptr_t* __restrict include_vec, const uintptr_t* __restrict male_vec, uint32_t* __restrict set_ctp, uint32_t* __restrict missing_ctp) {
uintptr_t loader = *lptr++;
uintptr_t loader2 = loader >> 1;
uintptr_t loader3 = *include_vec++;
uintptr_t loader4 = loader3 & (~(*male_vec));
uintptr_t set_odds = loader2 & loader4;
uintptr_t set_evens = loader & loader4;
uintptr_t missings_nm = set_evens & (~loader2);
uintptr_t missings_m;
uintptr_t males;
uintptr_t acc;
uintptr_t accm;
males = loader3 & (*male_vec++);
set_evens |= loader & loader2 & males;
missings_m = (loader ^ loader2) & males;
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
loader4 = loader3 & (~(*male_vec));
set_odds += loader2 & loader4;
loader4 &= loader;
set_evens += loader4;
missings_nm += loader4 & (~loader2);
loader4 = loader3 & (*male_vec++);
set_evens += loader & loader2 & loader4;
missings_m += (loader ^ loader2) & loader4;
males += loader4;
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
loader4 = loader3 & (~(*male_vec));
set_odds += loader2 & loader4;
loader4 &= loader;
set_evens += loader4;
missings_nm += loader4 & (~loader2);
loader4 = loader3 & (*male_vec++);
set_evens += loader & loader2 & loader4;
missings_m += (loader ^ loader2) & loader4;
males += loader4;
set_evens -= missings_nm;
set_odds = (set_odds & 0x33333333) + ((set_odds >> 2) & 0x33333333);
set_odds += (set_evens & 0x33333333) + ((set_evens >> 2) & 0x33333333);
missings_nm = ((missings_nm & 0x33333333) + ((missings_nm >> 2) & 0x33333333)) * 2;
missings_nm += (missings_m & 0x33333333) + ((missings_m >> 2) & 0x33333333);
missings_nm += (males & 0x33333333) + ((males >> 2) & 0x33333333);
acc = (set_odds & 0x0f0f0f0f) + ((set_odds >> 4) & 0x0f0f0f0f);
accm = (missings_nm & 0x0f0f0f0f) + ((missings_nm >> 4) & 0x0f0f0f0f);
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
loader4 = loader3 & (~(*male_vec));
set_odds = loader2 & loader4;
set_evens = loader & loader4;
missings_nm = set_evens & (~loader2);
males = loader3 & (*male_vec++);
set_evens |= loader & loader2 & males;
missings_m = (loader ^ loader2) & males;
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
loader4 = loader3 & (~(*male_vec));
set_odds += loader2 & loader4;
loader4 &= loader;
set_evens += loader4;
missings_nm += loader4 & (~loader2);
loader4 = loader3 & (*male_vec++);
set_evens += loader & loader2 & loader4;
missings_m += (loader ^ loader2) & loader4;
males += loader4;
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
loader4 = loader3 & (~(*male_vec));
set_odds += loader2 & loader4;
loader4 &= loader;
set_evens += loader4;
missings_nm += loader4 & (~loader2);
loader4 = loader3 & (*male_vec++);
set_evens += loader & loader2 & loader4;
missings_m += (loader ^ loader2) & loader4;
males += loader4;
set_evens -= missings_nm;
set_odds = (set_odds & 0x33333333) + ((set_odds >> 2) & 0x33333333);
set_odds += (set_evens & 0x33333333) + ((set_evens >> 2) & 0x33333333);
missings_nm = ((missings_nm & 0x33333333) + ((missings_nm >> 2) & 0x33333333)) * 2;
missings_nm += (missings_m & 0x33333333) + ((missings_m >> 2) & 0x33333333);
missings_nm += (males & 0x33333333) + ((males >> 2) & 0x33333333);
acc += (set_odds & 0x0f0f0f0f) + ((set_odds >> 4) & 0x0f0f0f0f);
accm += (missings_nm & 0x0f0f0f0f) + ((missings_nm >> 4) & 0x0f0f0f0f);
*set_ctp += (acc * 0x01010101) >> 24;
*missing_ctp += (accm * 0x01010101) >> 24;
}
void count_set_freq_y_12(const uintptr_t* __restrict lptr, const uintptr_t* __restrict include_vec, const uintptr_t* __restrict nonmale_vec, uint32_t* __restrict set_ctp, uint32_t* __restrict missing_ctp) {
uintptr_t loader = *lptr++;
uintptr_t loader2 = loader >> 1;
uintptr_t loader3 = *include_vec++;
uintptr_t loader4 = *nonmale_vec++;
uintptr_t sets1 = loader3 & loader & loader2 & (~loader4);
uintptr_t missings1 = loader3 & (loader4 | (loader ^ loader2));
uintptr_t sets2;
uintptr_t missings2;
uintptr_t acc;
uintptr_t accm;
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
loader4 = *nonmale_vec++;
sets1 += loader3 & loader & loader2 & (~loader4);
missings1 += loader3 & (loader4 | (loader ^ loader2));
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
loader4 = *nonmale_vec++;
sets1 += loader3 & loader & loader2 & (~loader4);
missings1 += loader3 & (loader4 | (loader ^ loader2));
sets1 = (sets1 & 0x33333333) + ((sets1 >> 2) & 0x33333333);
missings1 = (missings1 & 0x33333333) + ((missings1 >> 2) & 0x33333333);
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
loader4 = *nonmale_vec++;
sets2 = loader3 & loader & loader2 & (~loader4);
missings2 = loader3 & (loader4 | (loader ^ loader2));
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
loader4 = *nonmale_vec++;
sets2 += loader3 & loader & loader2 & (~loader4);
missings2 += loader3 & (loader4 | (loader ^ loader2));
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
loader4 = *nonmale_vec++;
sets2 += loader3 & loader & loader2 & (~loader4);
missings2 += loader3 & (loader4 | (loader ^ loader2));
sets1 += (sets2 & 0x33333333) + ((sets2 >> 2) & 0x33333333);
missings1 += (missings2 & 0x33333333) + ((missings2 >> 2) & 0x33333333);
acc = (sets1 & 0x0f0f0f0f) + ((sets1 >> 4) & 0x0f0f0f0f);
accm = (missings1 & 0x0f0f0f0f) + ((missings1 >> 4) & 0x0f0f0f0f);
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
loader4 = *nonmale_vec++;
sets1 = loader3 & loader & loader2 & (~loader4);
missings1 = loader3 & (loader4 | (loader ^ loader2));
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
loader4 = *nonmale_vec++;
sets1 += loader3 & loader & loader2 & (~loader4);
missings1 += loader3 & (loader4 | (loader ^ loader2));
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
loader4 = *nonmale_vec++;
sets1 += loader3 & loader & loader2 & (~loader4);
missings1 += loader3 & (loader4 | (loader ^ loader2));
sets1 = (sets1 & 0x33333333) + ((sets1 >> 2) & 0x33333333);
missings1 = (missings1 & 0x33333333) + ((missings1 >> 2) & 0x33333333);
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
loader4 = *nonmale_vec++;
sets2 = loader3 & loader & loader2 & (~loader4);
missings2 = loader3 & (loader4 | (loader ^ loader2));
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
loader4 = *nonmale_vec++;
sets2 += loader3 & loader & loader2 & (~loader4);
missings2 += loader3 & (loader4 | (loader ^ loader2));
loader = *lptr++;
loader2 = loader >> 1;
loader3 = *include_vec++;
loader4 = *nonmale_vec++;
sets2 += loader3 & loader & loader2 & (~loader4);
missings2 += loader3 & (loader4 | (loader ^ loader2));
sets1 += (sets2 & 0x33333333) + ((sets2 >> 2) & 0x33333333);
missings1 += (missings2 & 0x33333333) + ((missings2 >> 2) & 0x33333333);
acc += (sets1 & 0x0f0f0f0f) + ((sets1 >> 4) & 0x0f0f0f0f);
accm += (missings1 & 0x0f0f0f0f) + ((missings1 >> 4) & 0x0f0f0f0f);
*set_ctp += (acc * 0x01010101) >> 24;
*missing_ctp += (accm * 0x01010101) >> 24;
}
uintptr_t count_01_12(const uintptr_t* lptr) {
uintptr_t loader1 = *lptr++;
uintptr_t loader2 = *lptr++;
uintptr_t count1 = loader1 & (~(loader1 >> 1)) & FIVEMASK;
uintptr_t count2 = loader2 & (~(loader2 >> 1)) & FIVEMASK;
uintptr_t partial1;
uintptr_t partial2;
loader1 = *lptr++;
loader2 = *lptr++;
count1 += loader1 & (~(loader1 >> 1)) & FIVEMASK;
count2 += loader2 & (~(loader2 >> 1)) & FIVEMASK;
loader1 = *lptr++;
loader2 = *lptr++;
count1 += loader1 & (~(loader1 >> 1)) & FIVEMASK;
count2 += loader2 & (~(loader2 >> 1)) & FIVEMASK;
partial1 = (count1 & 0x33333333) + ((count1 >> 2) & 0x33333333);
partial2 = (count2 & 0x33333333) + ((count2 >> 2) & 0x33333333);
loader1 = *lptr++;
loader2 = *lptr++;
count1 = loader1 & (~(loader1 >> 1)) & FIVEMASK;
count2 = loader2 & (~(loader2 >> 1)) & FIVEMASK;
loader1 = *lptr++;
loader2 = *lptr++;
count1 += loader1 & (~(loader1 >> 1)) & FIVEMASK;
count2 += loader2 & (~(loader2 >> 1)) & FIVEMASK;
loader1 = *lptr++;
loader2 = *lptr++;
count1 += loader1 & (~(loader1 >> 1)) & FIVEMASK;
count2 += loader2 & (~(loader2 >> 1)) & FIVEMASK;
partial1 += (count1 & 0x33333333) + ((count1 >> 2) & 0x33333333);
partial2 += (count2 & 0x33333333) + ((count2 >> 2) & 0x33333333);
partial1 = (partial1 & 0x0f0f0f0f) + ((partial1 >> 4) & 0x0f0f0f0f);
partial1 += (partial2 & 0x0f0f0f0f) + ((partial2 >> 4) & 0x0f0f0f0f);
return (partial1 * 0x01010101) >> 24;
}
#endif
void genovec_set_freq(const uintptr_t* __restrict geno_vec, const uintptr_t* __restrict include_quatervec, uintptr_t sample_ctl2, uint32_t* __restrict set_ctp, uint32_t* __restrict missing_ctp) {
// Assuming include_quatervec describes e.g. cases, and an autosomal marker,
// this counts the number of case set alleles loaded in geno_vec[], as well
// as the number of cases with missing genotype info.
// See single_marker_freqs_and_hwe() for discussion.
// missing count: popcount2(genotype & (~(genotype >> 1)) & 0x5555...)
// set allele count: popcount(genotype) - missing count
const uintptr_t* geno_vec_end = &(geno_vec[sample_ctl2]);
uintptr_t loader;
uintptr_t loader2;
uintptr_t missing_incr;
uint32_t acc = 0;
uint32_t accm = 0;
#ifdef __LP64__
uintptr_t cur_decr = 60;
const uintptr_t* geno_vec_6x_end;
sample_ctl2 -= sample_ctl2 % 6;
while (sample_ctl2 >= 60) {
genovec_set_freq_loop:
geno_vec_6x_end = &(geno_vec[cur_decr]);
count_set_freq_60v((const __m128i*)geno_vec, (const __m128i*)geno_vec_6x_end, (const __m128i*)include_quatervec, &acc, &accm);
geno_vec = geno_vec_6x_end;
include_quatervec = &(include_quatervec[cur_decr]);
sample_ctl2 -= cur_decr;
}
if (sample_ctl2) {
cur_decr = sample_ctl2;
goto genovec_set_freq_loop;
}
#else
const uintptr_t* geno_vec_six_end = &(geno_vec[sample_ctl2 - (sample_ctl2 % 6)]);
while (geno_vec < geno_vec_six_end) {
count_set_freq_6(geno_vec, include_quatervec, &acc, &accm);
geno_vec = &(geno_vec[6]);
include_quatervec = &(include_quatervec[6]);
}
#endif
while (geno_vec < geno_vec_end) {
loader = *geno_vec++;
loader2 = *include_quatervec++;
missing_incr = popcount2_long(loader & (~(loader >> 1)) & loader2);
accm += missing_incr;
acc += popcount_long(loader & (loader2 * 3)) - missing_incr;
}
*set_ctp = acc;
*missing_ctp = accm;
}
void genovec_set_freq_x(const uintptr_t* __restrict geno_vec, const uintptr_t* __restrict include_quatervec, const uintptr_t* __restrict male_quatervec, uintptr_t sample_ctl2, uint32_t* __restrict set_ctp, uint32_t* __restrict missing_ctp) {
// diploid counting for nonmales, haploid counting for males
// missing_ct := male_obs + male_missing + 2 * female_missing
const uintptr_t* geno_vec_end = &(geno_vec[sample_ctl2]);
uintptr_t loader;
uintptr_t loader2;
uintptr_t loader3;
uintptr_t loader4;
uintptr_t missing_incr;
uint32_t acc = 0;
uint32_t accm = 0;
#ifdef __LP64__
uintptr_t cur_decr = 60;
const uintptr_t* geno_vec_6x_end;
sample_ctl2 -= sample_ctl2 % 6;
while (sample_ctl2 >= 60) {
genovec_set_freq_x_loop:
geno_vec_6x_end = &(geno_vec[cur_decr]);
count_set_freq_x_60v((const __m128i*)geno_vec, (const __m128i*)geno_vec_6x_end, (const __m128i*)include_quatervec, (const __m128i*)male_quatervec, &acc, &accm);
geno_vec = geno_vec_6x_end;
include_quatervec = &(include_quatervec[cur_decr]);
male_quatervec = &(male_quatervec[cur_decr]);
sample_ctl2 -= cur_decr;
}
if (sample_ctl2) {
cur_decr = sample_ctl2;
goto genovec_set_freq_x_loop;
}
#else
const uintptr_t* geno_vec_six_end = &(geno_vec[sample_ctl2 - (sample_ctl2 % 6)]);
while (geno_vec < geno_vec_six_end) {
count_set_freq_x_6(geno_vec, include_quatervec, male_quatervec, &acc, &accm);
geno_vec = &(geno_vec[6]);
include_quatervec = &(include_quatervec[6]);
male_quatervec = &(male_quatervec[6]);
}
#endif
while (geno_vec < geno_vec_end) {
loader = *geno_vec++;
loader2 = loader >> 1;
loader3 = *include_quatervec++;
loader4 = loader3 & (~(*male_quatervec));
missing_incr = popcount2_long(loader & (~loader2) & loader4);
accm += 2 * missing_incr;
acc += popcount_long(loader & (loader4 * 3)) - missing_incr;
loader4 = loader3 & (*male_quatervec++);
acc += popcount2_long(loader & loader2 & loader4);
accm += popcount_long(((loader ^ loader2) & loader4) | (loader4 << 1));
}
*set_ctp = acc;
*missing_ctp = accm;
}
void genovec_set_freq_y(const uintptr_t* __restrict geno_vec, const uintptr_t* __restrict include_quatervec, const uintptr_t* __restrict nonmale_quatervec, uintptr_t sample_ctl2, uint32_t* __restrict set_ctp, uint32_t* __restrict missing_ctp) {
// all nonmales contribute to missing_ct here
const uintptr_t* geno_vec_end = &(geno_vec[sample_ctl2]);
uintptr_t loader;
uintptr_t loader2;
uintptr_t loader3;
uintptr_t loader4;
uint32_t acc = 0;
uint32_t accm = 0;
#ifdef __LP64__
uintptr_t cur_decr = 120;
const uintptr_t* geno_vec_12x_end;
sample_ctl2 -= sample_ctl2 % 12;
while (sample_ctl2 >= 120) {
genovec_set_freq_y_loop:
geno_vec_12x_end = &(geno_vec[cur_decr]);
count_set_freq_y_120v((__m128i*)geno_vec, (__m128i*)geno_vec_12x_end, (__m128i*)include_quatervec, (__m128i*)nonmale_quatervec, &acc, &accm);
geno_vec = geno_vec_12x_end;
include_quatervec = &(include_quatervec[cur_decr]);
nonmale_quatervec = &(nonmale_quatervec[cur_decr]);
sample_ctl2 -= cur_decr;
}
if (sample_ctl2) {
cur_decr = sample_ctl2;
goto genovec_set_freq_y_loop;
}
#else
const uintptr_t* geno_vec_twelve_end = &(geno_vec[sample_ctl2 - (sample_ctl2 % 12)]);
while (geno_vec < geno_vec_twelve_end) {
count_set_freq_y_12(geno_vec, include_quatervec, nonmale_quatervec, &acc, &accm);
geno_vec = &(geno_vec[12]);
include_quatervec = &(include_quatervec[12]);
nonmale_quatervec = &(nonmale_quatervec[12]);
}
#endif
while (geno_vec < geno_vec_end) {
loader = *geno_vec++;
loader2 = loader >> 1;
loader3 = *include_quatervec++;
loader4 = *nonmale_quatervec++;
acc += popcount2_long(loader & loader2 & loader3 & (~loader4));
accm += popcount2_long(loader3 & ((loader ^ loader2) | loader4));
}
*set_ctp = acc;
*missing_ctp = accm;
}
void genovec_3freq(const uintptr_t* __restrict geno_vec, const uintptr_t* __restrict include_quatervec, uintptr_t sample_ctl2, uint32_t* __restrict missing_ctp, uint32_t* __restrict het_ctp, uint32_t* __restrict homset_ctp) {
// generic routine for getting all counts.
const uintptr_t* geno_vec_end = &(geno_vec[sample_ctl2]);
uintptr_t loader;
uintptr_t loader2;
uintptr_t loader3;
uint32_t acc_even = 0;
uint32_t acc_odd = 0;
uint32_t acc_and = 0;
#ifdef __LP64__
uintptr_t cur_decr = 120;
const uintptr_t* geno_vec_12x_end;
sample_ctl2 -= sample_ctl2 % 12;
while (sample_ctl2 >= 120) {
genovec_3freq_loop:
geno_vec_12x_end = &(geno_vec[cur_decr]);
count_3freq_1920b((const __m128i*)geno_vec, (const __m128i*)geno_vec_12x_end, (const __m128i*)include_quatervec, &acc_even, &acc_odd, &acc_and);
geno_vec = geno_vec_12x_end;
include_quatervec = &(include_quatervec[cur_decr]);
sample_ctl2 -= cur_decr;
}
if (sample_ctl2) {
cur_decr = sample_ctl2;
goto genovec_3freq_loop;
}
#else
const uintptr_t* geno_vec_twelve_end = &(geno_vec[sample_ctl2 - (sample_ctl2 % 12)]);
while (geno_vec < geno_vec_twelve_end) {
count_3freq_48b(geno_vec, include_quatervec, &acc_even, &acc_odd, &acc_and);
geno_vec = &(geno_vec[12]);
include_quatervec = &(include_quatervec[12]);
}
#endif
while (geno_vec < geno_vec_end) {
loader = *geno_vec++;
loader2 = *include_quatervec++;
loader3 = loader2 & (loader >> 1);
acc_even += popcount2_long(loader & loader2);
acc_odd += popcount2_long(loader3);
acc_and += popcount2_long(loader & loader3);
}
*missing_ctp = acc_even - acc_and;
*het_ctp = acc_odd - acc_and;
*homset_ctp = acc_and;
}
uintptr_t count_01(const uintptr_t* quatervec, uintptr_t word_ct) {
// really just for getting a missing count
// unlike popcount01_longs, this does not assume quatervec[] has no 11s
const uintptr_t* quatervec_end = &(quatervec[word_ct]);
uintptr_t loader;
#ifdef __LP64__
uintptr_t acc;
word_ct -= word_ct % 12;
acc = count_01_vecs((__m128i*)quatervec, word_ct / 2);
quatervec = &(quatervec[word_ct]);
#else
const uintptr_t* quatervec_twelve_end = &(quatervec[word_ct - (word_ct % 12)]);
uintptr_t acc = 0;
while (quatervec < quatervec_twelve_end) {
acc += count_01_12(quatervec);
quatervec = &(quatervec[12]);
}
#endif
while (quatervec < quatervec_end) {
loader = *quatervec++;
acc += popcount2_long(loader & (~(loader >> 1)) & FIVEMASK);
}
return acc;
}
void fill_all_bits(uintptr_t ct, uintptr_t* bitarr) {
// leaves bits beyond the end unset
// ok for ct == 0
uintptr_t quotient = ct / BITCT;
uintptr_t remainder = ct % BITCT;
fill_ulong_one(quotient, bitarr);
if (remainder) {
bitarr[quotient] = (ONELU << remainder) - ONELU;
}
}
uint32_t numeric_range_list_to_bitarr(const Range_list* range_list_ptr, uint32_t item_ct, uint32_t offset, uint32_t ignore_overflow, uintptr_t* bitarr) {
// bitarr assumed to be initialized
const char* names = range_list_ptr->names;
const unsigned char* starts_range = range_list_ptr->starts_range;
uint32_t name_ct = range_list_ptr->name_ct;
uint32_t name_max_len = range_list_ptr->name_max_len;
uint32_t idx_max = item_ct + offset;
uint32_t name_idx;
uint32_t idx1;
uint32_t idx2;
for (name_idx = 0; name_idx < name_ct; name_idx++) {
if (scan_uint_capped(&(names[name_idx * name_max_len]), idx_max, &idx1)) {
if (ignore_overflow) {
continue;
}
return 1;
}
if (starts_range[name_idx]) {
name_idx++;
if (scan_uint_capped(&(names[name_idx * name_max_len]), idx_max, &idx2)) {
if (!ignore_overflow) {
return 1;
}
idx2 = idx_max - 1;
}
fill_bits(idx1 - offset, (idx2 - idx1) + 1, bitarr);
} else {
set_bit(idx1 - offset, bitarr);
}
}
return 0;
}
int32_t string_range_list_to_bitarr(char* header_line, uint32_t item_ct, uint32_t fixed_len, const Range_list* range_list_ptr, const char* __restrict sorted_ids, const uint32_t* __restrict id_map, const char* __restrict range_list_flag, const char* __restrict file_descrip, uintptr_t* bitarr, int32_t* __restrict seen_idxs) {
// bitarr assumed to be initialized
// if fixed_len is zero, header_line is assumed to be a list of
// space-delimited unequal-length names
uintptr_t max_id_len = range_list_ptr->name_max_len;
uintptr_t name_ct = range_list_ptr->name_ct;
uint32_t item_idx = 0;
int32_t retval = 0;
char* bufptr;
uint32_t cmdline_pos;
int32_t ii;
while (1) {
bufptr = token_endnn(header_line);
ii = bsearch_str(header_line, (uintptr_t)(bufptr - header_line), sorted_ids, max_id_len, name_ct);
if (ii != -1) {
cmdline_pos = id_map[(uint32_t)ii];
if (seen_idxs[cmdline_pos] != -1) {
snprintf(g_logbuf, LOGBUFLEN, "Error: Duplicate --%s token in %s.\n", range_list_flag, file_descrip);
goto string_range_list_to_bitarr_ret_INVALID_FORMAT_2;
}
seen_idxs[cmdline_pos] = item_idx;
if (cmdline_pos && range_list_ptr->starts_range[cmdline_pos - 1]) {
if (seen_idxs[cmdline_pos - 1] == -1) {
LOGPREPRINTFWW("Error: Second element of --%s range appears before first element in %s.\n", range_list_flag, file_descrip);
goto string_range_list_to_bitarr_ret_INVALID_CMDLINE_2;
}
fill_bits(seen_idxs[cmdline_pos - 1], (item_idx - seen_idxs[cmdline_pos - 1]) + 1, bitarr);
} else if (!(range_list_ptr->starts_range[cmdline_pos])) {
SET_BIT(item_idx, bitarr);
}
}
if (++item_idx == item_ct) {
break;
}
if (fixed_len) {
header_line = &(header_line[fixed_len]);
} else {
header_line = skip_initial_spaces(&(bufptr[1]));
}
}
for (cmdline_pos = 0; cmdline_pos < name_ct; cmdline_pos++) {
if (seen_idxs[cmdline_pos] == -1) {
goto string_range_list_to_bitarr_ret_INVALID_CMDLINE_3;
}
}
while (0) {
string_range_list_to_bitarr_ret_INVALID_CMDLINE_3:
snprintf(g_logbuf, LOGBUFLEN, "Error: Missing --%s token in %s.\n", range_list_flag, file_descrip);
string_range_list_to_bitarr_ret_INVALID_CMDLINE_2:
logerrprintb();
retval = RET_INVALID_CMDLINE;
break;
string_range_list_to_bitarr_ret_INVALID_FORMAT_2:
logerrprintb();
retval = RET_INVALID_FORMAT;
break;
}
return retval;
}
int32_t string_range_list_to_bitarr_alloc(char* header_line, uint32_t item_ct, uint32_t fixed_len, const Range_list* range_list_ptr, const char* __restrict range_list_flag, const char* __restrict file_descrip, uintptr_t** bitarr_ptr) {
// wrapper for string_range_list_to_bitarr which allocates the bitfield and
// temporary buffers on the heap
uintptr_t item_ctl = BITCT_TO_WORDCT(item_ct);
uintptr_t name_ct = range_list_ptr->name_ct;
int32_t retval = 0;
int32_t* seen_idxs;
char* sorted_ids;
uint32_t* id_map;
if (bigstack_calloc_ul(item_ctl, bitarr_ptr) ||
bigstack_alloc_i(name_ct, &seen_idxs)) {
return RET_NOMEM;
}
// kludge to use sort_item_ids()
fill_ulong_zero(BITCT_TO_WORDCT(name_ct), (uintptr_t*)seen_idxs);
if (sort_item_ids(name_ct, (uintptr_t*)seen_idxs, 0, range_list_ptr->names, range_list_ptr->name_max_len, 0, 0, strcmp_deref, &sorted_ids, &id_map)) {
return RET_NOMEM;
}
fill_int_one(name_ct, seen_idxs);
retval = string_range_list_to_bitarr(header_line, item_ct, fixed_len, range_list_ptr, sorted_ids, id_map, range_list_flag, file_descrip, *bitarr_ptr, seen_idxs);
bigstack_reset(seen_idxs);
return retval;
}
int32_t string_range_list_to_bitarr2(const char* __restrict sorted_ids, const uint32_t* id_map, uintptr_t item_ct, uintptr_t max_id_len, const Range_list* __restrict range_list_ptr, const char* __restrict range_list_flag, uintptr_t* bitfield_excl) {
// sorted_ids/id_map is for e.g. marker IDs instead of command line
// parameters. bitfield_excl is assumed to be initialized (since its length
// is not known by this function).
char* names = range_list_ptr->names;
const unsigned char* starts_range = range_list_ptr->starts_range;
uintptr_t name_max_len = range_list_ptr->name_max_len;
uint32_t name_ct = range_list_ptr->name_ct;
int32_t retval = 0;
uint32_t param_idx;
char* bufptr;
uint32_t item_uidx;
uint32_t item_uidx2;
int32_t ii;
for (param_idx = 0; param_idx < name_ct; param_idx++) {
bufptr = &(names[param_idx * name_max_len]);
ii = bsearch_str_nl(bufptr, sorted_ids, max_id_len, item_ct);
if (ii == -1) {
goto string_range_list_to_bitarr2_ret_INVALID_CMDLINE_3;
}
item_uidx = id_map[(uint32_t)ii];
if (starts_range[param_idx]) {
param_idx++;
bufptr = &(names[param_idx * name_max_len]);
ii = bsearch_str_nl(bufptr, sorted_ids, max_id_len, item_ct);
if (ii == -1) {
goto string_range_list_to_bitarr2_ret_INVALID_CMDLINE_3;
}
item_uidx2 = id_map[(uint32_t)ii];
if (item_uidx2 < item_uidx) {
snprintf(g_logbuf, LOGBUFLEN, "Error: Second element of --%s range appears before first.\n", range_list_flag);
goto string_range_list_to_bitarr2_ret_INVALID_CMDLINE_2;
}
clear_bits(item_uidx, item_uidx2 - item_uidx + 1, bitfield_excl);
} else {
clear_bit(item_uidx, bitfield_excl);
}
}
while (0) {
string_range_list_to_bitarr2_ret_INVALID_CMDLINE_3:
snprintf(g_logbuf, LOGBUFLEN, "Error: --%s ID not found.\n", range_list_flag);
string_range_list_to_bitarr2_ret_INVALID_CMDLINE_2:
logerrprintb();
retval = RET_INVALID_CMDLINE;
break;
}
return retval;
}
uint32_t count_non_autosomal_markers(const Chrom_info* chrom_info_ptr, const uintptr_t* marker_exclude, uint32_t count_x, uint32_t count_mt) {
// for backward compatibility, unplaced markers are considered to be
// autosomal here
const int32_t x_code = chrom_info_ptr->xymt_codes[X_OFFSET];
const int32_t y_code = chrom_info_ptr->xymt_codes[Y_OFFSET];
const int32_t mt_code = chrom_info_ptr->xymt_codes[MT_OFFSET];
uint32_t ct = 0;
if (count_x && (x_code != -2)) {
ct += count_chrom_markers(chrom_info_ptr, marker_exclude, x_code);
}
if (y_code != -2) {
ct += count_chrom_markers(chrom_info_ptr, marker_exclude, y_code);
}
if (count_mt && (mt_code != -2)) {
ct += count_chrom_markers(chrom_info_ptr, marker_exclude, mt_code);
}
return ct;
}
int32_t conditional_allocate_non_autosomal_markers(const Chrom_info* chrom_info_ptr, uintptr_t unfiltered_marker_ct, const uintptr_t* marker_exclude_orig, uint32_t marker_ct, uint32_t count_x, uint32_t count_mt, const char* calc_descrip, uintptr_t** marker_exclude_ptr, uint32_t* newly_excluded_ct_ptr) {
// if all markers are autosomal (or pseudoautosomal) diploid, nothing
// happens. otherwise, this creates a marker_exclude copy with
// non-{autosomal diploid} markers excluded for the caller to use.
const uintptr_t unfiltered_marker_ctl = BITCT_TO_WORDCT(unfiltered_marker_ct);
const int32_t* xymt_codes = chrom_info_ptr->xymt_codes;
uint32_t xymt_cts[XYMT_OFFSET_CT];
fill_uint_zero(XYMT_OFFSET_CT, xymt_cts);
if (is_set(chrom_info_ptr->haploid_mask, 0)) {
*newly_excluded_ct_ptr = marker_ct;
} else {
if (count_x && (xymt_codes[X_OFFSET] != -2)) {
xymt_cts[X_OFFSET] = count_chrom_markers(chrom_info_ptr, marker_exclude_orig, xymt_codes[X_OFFSET]);
}
if (xymt_codes[Y_OFFSET] != -2) {
xymt_cts[Y_OFFSET] = count_chrom_markers(chrom_info_ptr, marker_exclude_orig, xymt_codes[Y_OFFSET]);
}
if (count_mt && (xymt_codes[MT_OFFSET] != -2)) {
xymt_cts[MT_OFFSET] = count_chrom_markers(chrom_info_ptr, marker_exclude_orig, xymt_codes[MT_OFFSET]);
}
*newly_excluded_ct_ptr = xymt_cts[X_OFFSET] + xymt_cts[Y_OFFSET] + xymt_cts[MT_OFFSET];
}
if (*newly_excluded_ct_ptr) {
LOGPRINTF("Excluding %u variant%s on non-autosomes from %s.\n", *newly_excluded_ct_ptr, (*newly_excluded_ct_ptr == 1)? "" : "s", calc_descrip);
}
if (*newly_excluded_ct_ptr == marker_ct) {
logerrprint("Error: No variants remaining.\n");
return RET_INVALID_CMDLINE;
}
if (!(*newly_excluded_ct_ptr)) {
return 0;
}
if (bigstack_alloc_ul(unfiltered_marker_ctl, marker_exclude_ptr)) {
return RET_NOMEM;
}
memcpy(*marker_exclude_ptr, marker_exclude_orig, unfiltered_marker_ctl * sizeof(intptr_t));
for (uint32_t xymt_idx = 0; xymt_idx < XYMT_OFFSET_CT; ++xymt_idx) {
if (xymt_cts[xymt_idx]) {
const uint32_t chrom_fo_idx = chrom_info_ptr->chrom_idx_to_foidx[xymt_codes[xymt_idx]];
fill_bits(chrom_info_ptr->chrom_fo_vidx_start[chrom_fo_idx], chrom_info_ptr->chrom_fo_vidx_start[chrom_fo_idx + 1] - chrom_info_ptr->chrom_fo_vidx_start[chrom_fo_idx], *marker_exclude_ptr);
}
}
return 0;
}
uint32_t get_max_chrom_size(const Chrom_info* chrom_info_ptr, const uintptr_t* marker_exclude, uint32_t* last_chrom_fo_idx_ptr) {
const uint32_t chrom_ct = chrom_info_ptr->chrom_ct;
uint32_t max_chrom_size = 0;
uint32_t last_chrom_fo_idx = 0;
for (uint32_t chrom_fo_idx = 0; chrom_fo_idx < chrom_ct; chrom_fo_idx++) {
const uint32_t cur_chrom_size = count_chrom_markers(chrom_info_ptr, marker_exclude, chrom_info_ptr->chrom_file_order[chrom_fo_idx]);
if (cur_chrom_size) {
last_chrom_fo_idx = chrom_fo_idx;
if (cur_chrom_size > max_chrom_size) {
max_chrom_size = cur_chrom_size;
}
}
}
if (last_chrom_fo_idx_ptr) {
*last_chrom_fo_idx_ptr = last_chrom_fo_idx;
}
return max_chrom_size;
}
void count_genders(const uintptr_t* __restrict sex_nm, const uintptr_t* __restrict sex_male, const uintptr_t* __restrict sample_exclude, uintptr_t unfiltered_sample_ct, uint32_t* __restrict male_ct_ptr, uint32_t* __restrict female_ct_ptr, uint32_t* __restrict unk_ct_ptr) {
// unfiltered_sample_ct can be zero
uint32_t male_ct = 0;
uint32_t female_ct = 0;
uint32_t unk_ct = 0;
uint32_t unfiltered_sample_ctld = unfiltered_sample_ct / BITCT;
uint32_t unfiltered_sample_ct_rem = unfiltered_sample_ct & (BITCT - 1);
uintptr_t ulii;
uintptr_t uljj;
uintptr_t sample_bidx;
for (sample_bidx = 0; sample_bidx < unfiltered_sample_ctld; sample_bidx++) {
ulii = ~(*sample_exclude++);
count_genders_last_loop:
uljj = *sex_nm++;
unk_ct += popcount_long(ulii & (~uljj));
ulii &= uljj;
uljj = *sex_male++;
male_ct += popcount_long(ulii & uljj);
female_ct += popcount_long(ulii & (~uljj));
}
if (unfiltered_sample_ct_rem) {
ulii = (~(*sample_exclude)) & ((ONELU << unfiltered_sample_ct_rem) - ONELU);
unfiltered_sample_ct_rem = 0;
goto count_genders_last_loop;
}
*male_ct_ptr = male_ct;
*female_ct_ptr = female_ct;
*unk_ct_ptr = unk_ct;
}
void reverse_loadbuf(uintptr_t unfiltered_sample_ct, unsigned char* loadbuf) {
// unfiltered_sample_ct can be zero
uintptr_t sample_bidx = 0;
unsigned char* loadbuf_end = &(loadbuf[(unfiltered_sample_ct + 3) / 4]);
unsigned char ucc;
unsigned char ucc2;
uintptr_t unfiltered_sample_ctd;
uint32_t* loadbuf_alias32;
uint32_t uii;
uint32_t ujj;
#ifdef __LP64__
const __m128i m1 = {FIVEMASK, FIVEMASK};
__m128i* loadbuf_alias;
__m128i vii;
__m128i vjj;
// todo: use this vector loop even when loadbuf is unaligned, so stuff like
// recode_load_to() is faster
if (!(((uintptr_t)loadbuf) & 15)) {
loadbuf_alias = (__m128i*)loadbuf;
unfiltered_sample_ctd = unfiltered_sample_ct / 64;
for (; sample_bidx < unfiltered_sample_ctd; sample_bidx++) {
vii = *loadbuf_alias;
// we want to exchange 00 and 11, and leave 01/10 untouched. So make
// vjj := 11 iff vii is 00/11, and vjj := 00 otherwise; then xor.
vjj = _mm_andnot_si128(_mm_xor_si128(vii, _mm_srli_epi64(vii, 1)), m1);
vjj = _mm_or_si128(vjj, _mm_slli_epi64(vjj, 1));
*loadbuf_alias++ = _mm_xor_si128(vii, vjj);
}
loadbuf = (unsigned char*)loadbuf_alias;
} else if (!(((uintptr_t)loadbuf) & 3)) {
loadbuf_alias32 = (uint32_t*)loadbuf;
unfiltered_sample_ctd = unfiltered_sample_ct / BITCT2;
for (; sample_bidx < unfiltered_sample_ctd; sample_bidx++) {
uii = *loadbuf_alias32;
ujj = 0x55555555 & (~(uii ^ (uii >> 1)));
ujj *= 3;
*loadbuf_alias32++ = uii ^ ujj;
}
loadbuf = (unsigned char*)loadbuf_alias32;
}
#else
if (!(((uintptr_t)loadbuf) & 3)) {
loadbuf_alias32 = (uint32_t*)loadbuf;
unfiltered_sample_ctd = unfiltered_sample_ct / BITCT2;
for (; sample_bidx < unfiltered_sample_ctd; sample_bidx++) {
uii = *loadbuf_alias32;
ujj = 0x55555555 & (~(uii ^ (uii >> 1)));
ujj *= 3;
*loadbuf_alias32++ = uii ^ ujj;
}
loadbuf = (unsigned char*)loadbuf_alias32;
}
#endif
for (; loadbuf < loadbuf_end;) {
ucc = *loadbuf;
ucc2 = 0x55 & (~(ucc ^ (ucc >> 1)));
ucc2 *= 3;
*loadbuf++ = ucc ^ ucc2;
}
uii = unfiltered_sample_ct & 3;
if (uii) {
loadbuf[-1] &= (0xff >> (8 - 2 * uii));
}
}
// deprecated, try to just use copy_quaterarr_nonempty_subset()
void copy_quaterarr_nonempty_subset_excl(const uintptr_t* __restrict raw_quaterarr, const uintptr_t* __restrict subset_excl, uint32_t raw_quaterarr_size, uint32_t subset_size, uintptr_t* __restrict output_quaterarr) {
assert(subset_size);
assert(raw_quaterarr_size >= subset_size);
uintptr_t cur_output_word = 0;
uintptr_t* output_quaterarr_last = &(output_quaterarr[subset_size / BITCT2]);
const uint32_t word_write_halfshift_end = subset_size % BITCT2;
uint32_t word_write_halfshift = 0;
// if < 2/3-filled, use sparse copy algorithm
if (subset_size * (3 * ONELU) < raw_quaterarr_size * (2 * ONELU)) {
const uint32_t subset_excl_widx_last = raw_quaterarr_size / BITCT;
uint32_t subset_excl_widx = 0;
while (1) {
uintptr_t cur_include_word = ~subset_excl[subset_excl_widx];
// this, kiddies, is why exclude masks were a mistake.
if (subset_excl_widx == subset_excl_widx_last) {
cur_include_word &= (ONELU << (raw_quaterarr_size % BITCT)) - ONELU;
}
if (cur_include_word) {
uint32_t wordhalf_idx = 0;
#ifdef __LP64__
uint32_t cur_include_halfword = (uint32_t)cur_include_word;
#else
uint32_t cur_include_halfword = (uint16_t)cur_include_word;
#endif
while (1) {
if (cur_include_halfword) {
uintptr_t raw_quaterarr_word = raw_quaterarr[subset_excl_widx * 2 + wordhalf_idx];
do {
uint32_t rqa_idx_lowbits = __builtin_ctz(cur_include_halfword);
cur_output_word |= ((raw_quaterarr_word >> (rqa_idx_lowbits * 2)) & 3) << (word_write_halfshift * 2);
if (++word_write_halfshift == BITCT2) {
*output_quaterarr++ = cur_output_word;
word_write_halfshift = 0;
cur_output_word = 0;
}
cur_include_halfword &= cur_include_halfword - 1;
} while (cur_include_halfword);
}
if (wordhalf_idx) {
break;
}
wordhalf_idx++;
#ifdef __LP64__
cur_include_halfword = cur_include_word >> 32;
#else
cur_include_halfword = cur_include_word >> 16;
#endif
}
if (output_quaterarr == output_quaterarr_last) {
if (word_write_halfshift == word_write_halfshift_end) {
if (word_write_halfshift_end) {
*output_quaterarr_last = cur_output_word;
}
return;
}
}
}
subset_excl_widx++;
}
}
// blocked copy
const uintptr_t* subset_excl_last = &(subset_excl[raw_quaterarr_size / BITCT]);
while (1) {
uintptr_t cur_include_word = ~(*subset_excl);
if (subset_excl == subset_excl_last) {
cur_include_word &= (ONELU << (raw_quaterarr_size % BITCT)) - ONELU;
}
subset_excl++;
uint32_t wordhalf_idx = 0;
#ifdef __LP64__
uintptr_t cur_include_halfword = (uint32_t)cur_include_word;
#else
uint32_t cur_include_halfword = (uint16_t)cur_include_word;
#endif
while (1) {
uintptr_t raw_quaterarr_word = *raw_quaterarr++;
while (cur_include_halfword) {
uint32_t rqa_idx_lowbits = CTZLU(cur_include_halfword);
uintptr_t halfword_invshifted = (~cur_include_halfword) >> rqa_idx_lowbits;
uintptr_t raw_quaterarr_curblock_unmasked = raw_quaterarr_word >> (rqa_idx_lowbits * 2);
uint32_t rqa_block_len = CTZLU(halfword_invshifted);
uint32_t block_len_limit = BITCT2 - word_write_halfshift;
cur_output_word |= raw_quaterarr_curblock_unmasked << (2 * word_write_halfshift);
if (rqa_block_len < block_len_limit) {
word_write_halfshift += rqa_block_len;
cur_output_word &= (ONELU << (word_write_halfshift * 2)) - ONELU;
} else {
// no need to mask, extra bits vanish off the high end
*output_quaterarr++ = cur_output_word;
word_write_halfshift = rqa_block_len - block_len_limit;
cur_output_word = (raw_quaterarr_curblock_unmasked >> (2 * block_len_limit)) & ((ONELU << (2 * word_write_halfshift)) - ONELU);
}
cur_include_halfword &= (~(ONELU << (rqa_block_len + rqa_idx_lowbits))) + ONELU;
}
if (wordhalf_idx) {
break;
}
wordhalf_idx++;
#ifdef __LP64__
cur_include_halfword = cur_include_word >> 32;
#else
cur_include_halfword = cur_include_word >> 16;
#endif
}
if (output_quaterarr == output_quaterarr_last) {
if (word_write_halfshift == word_write_halfshift_end) {
if (word_write_halfshift_end) {
*output_quaterarr_last = cur_output_word;
}
return;
}
}
}
}
uint32_t load_and_collapse(uint32_t unfiltered_sample_ct, uint32_t sample_ct, const uintptr_t* __restrict sample_exclude, uintptr_t final_mask, uint32_t do_reverse, FILE* bedfile, uintptr_t* __restrict rawbuf, uintptr_t* __restrict mainbuf) {
assert(unfiltered_sample_ct);
uint32_t unfiltered_sample_ct4 = (unfiltered_sample_ct + 3) / 4;
if (unfiltered_sample_ct == sample_ct) {
rawbuf = mainbuf;
}
if (load_raw(unfiltered_sample_ct4, bedfile, rawbuf)) {
return RET_READ_FAIL;
}
if (unfiltered_sample_ct != sample_ct) {
copy_quaterarr_nonempty_subset_excl(rawbuf, sample_exclude, unfiltered_sample_ct, sample_ct, mainbuf);
} else {
rawbuf[(unfiltered_sample_ct - 1) / BITCT2] &= final_mask;
}
if (do_reverse) {
reverse_loadbuf(sample_ct, (unsigned char*)mainbuf);
}
return 0;
}
void copy_quaterarr_nonempty_subset(const uintptr_t* __restrict raw_quaterarr, const uintptr_t* __restrict subset_mask, uint32_t raw_quaterarr_size, uint32_t subset_size, uintptr_t* __restrict output_quaterarr) {
// in plink 2.0, we probably want (0-based) bit raw_quaterarr_size of
// subset_mask to be always allocated and unset. This removes a few special
// cases re: iterating past the end of arrays.
assert(subset_size);
assert(raw_quaterarr_size >= subset_size);
uintptr_t cur_output_word = 0;
uintptr_t* output_quaterarr_last = &(output_quaterarr[subset_size / BITCT2]);
const uint32_t word_write_halfshift_end = subset_size % BITCT2;
uint32_t word_write_halfshift = 0;
// if < 2/3-filled, use sparse copy algorithm
if (subset_size * (3 * ONELU) < raw_quaterarr_size * (2 * ONELU)) {
uint32_t subset_mask_widx = 0;
while (1) {
const uintptr_t cur_include_word = subset_mask[subset_mask_widx];
if (cur_include_word) {
uint32_t wordhalf_idx = 0;
#ifdef __LP64__
uint32_t cur_include_halfword = (uint32_t)cur_include_word;
#else
uint32_t cur_include_halfword = (uint16_t)cur_include_word;
#endif
while (1) {
if (cur_include_halfword) {
uintptr_t raw_quaterarr_word = raw_quaterarr[subset_mask_widx * 2 + wordhalf_idx];
do {
uint32_t rqa_idx_lowbits = __builtin_ctz(cur_include_halfword);
cur_output_word |= ((raw_quaterarr_word >> (rqa_idx_lowbits * 2)) & 3) << (word_write_halfshift * 2);
if (++word_write_halfshift == BITCT2) {
*output_quaterarr++ = cur_output_word;
word_write_halfshift = 0;
cur_output_word = 0;
}
cur_include_halfword &= cur_include_halfword - 1;
} while (cur_include_halfword);
}
if (wordhalf_idx) {
break;
}
wordhalf_idx++;
#ifdef __LP64__
cur_include_halfword = cur_include_word >> 32;
#else
cur_include_halfword = cur_include_word >> 16;
#endif
}
if (output_quaterarr == output_quaterarr_last) {
if (word_write_halfshift == word_write_halfshift_end) {
if (word_write_halfshift_end) {
*output_quaterarr_last = cur_output_word;
}
return;
}
}
}
subset_mask_widx++;
}
}
// blocked copy
while (1) {
const uintptr_t cur_include_word = *subset_mask++;
uint32_t wordhalf_idx = 0;
#ifdef __LP64__
uintptr_t cur_include_halfword = (uint32_t)cur_include_word;
#else
uint32_t cur_include_halfword = (uint16_t)cur_include_word;
#endif
while (1) {
uintptr_t raw_quaterarr_word = *raw_quaterarr++;
while (cur_include_halfword) {
uint32_t rqa_idx_lowbits = CTZLU(cur_include_halfword);
uintptr_t halfword_invshifted = (~cur_include_halfword) >> rqa_idx_lowbits;
uintptr_t raw_quaterarr_curblock_unmasked = raw_quaterarr_word >> (rqa_idx_lowbits * 2);
uint32_t rqa_block_len = CTZLU(halfword_invshifted);
uint32_t block_len_limit = BITCT2 - word_write_halfshift;
cur_output_word |= raw_quaterarr_curblock_unmasked << (2 * word_write_halfshift);
if (rqa_block_len < block_len_limit) {
word_write_halfshift += rqa_block_len;
cur_output_word &= (ONELU << (word_write_halfshift * 2)) - ONELU;
} else {
// no need to mask, extra bits vanish off the high end
*output_quaterarr++ = cur_output_word;
word_write_halfshift = rqa_block_len - block_len_limit;
if (word_write_halfshift) {
cur_output_word = (raw_quaterarr_curblock_unmasked >> (2 * block_len_limit)) & ((ONELU << (2 * word_write_halfshift)) - ONELU);
} else {
// avoid potential right-shift-64
cur_output_word = 0;
}
}
cur_include_halfword &= (~(ONELU << (rqa_block_len + rqa_idx_lowbits))) + ONELU;
}
if (wordhalf_idx) {
break;
}
wordhalf_idx++;
#ifdef __LP64__
cur_include_halfword = cur_include_word >> 32;
#else
cur_include_halfword = cur_include_word >> 16;
#endif
}
if (output_quaterarr == output_quaterarr_last) {
if (word_write_halfshift == word_write_halfshift_end) {
if (word_write_halfshift_end) {
*output_quaterarr_last = cur_output_word;
}
return;
}
}
}
}
/*
void inplace_quaterarr_proper_subset(const uintptr_t* __restrict subset_mask, uint32_t orig_quaterarr_size, uint32_t subset_size, uintptr_t* __restrict main_quaterarr) {
assert(orig_quaterarr_size > subset_size);
// worthwhile to special-case this since we get to entirely skip
// reading/writing these words
if (!(~subset_mask[0])) {
const uintptr_t* subset_mask_initial = subset_mask;
// guaranteed to terminate since orig_quaterarr_size > subset_size.
do {
subset_mask++;
} while (!(~subset_mask[0]));
const uint32_t quaterarr_word_skip_ct = 2 * ((uintptr_t)(subset_mask - subset_mask_initial));
main_quaterarr = &(main_quaterarr[quaterarr_word_skip_ct]);
const uint32_t item_skip_ct = quaterarr_word_skip_ct * BITCT2;
orig_quaterarr_size -= item_skip_ct;
subset_size -= item_skip_ct;
}
uintptr_t cur_output_word = 0;
uintptr_t* main_quaterarr_writer = main_quaterarr;
uintptr_t* main_quaterarr_write_last = &(main_quaterarr[subset_size / BITCT2]);
const uint32_t word_write_halfshift_end = subset_size % BITCT2;
uint32_t word_write_halfshift = 0;
// if <= 2/3-filled, use sparse copy algorithm
if (subset_size * (3 * ONELU) <= orig_quaterarr_size * (2 * ONELU)) {
uint32_t subset_mask_widx = 0;
while (1) {
const uintptr_t cur_include_word = subset_mask[subset_mask_widx];
if (cur_include_word) {
uint32_t wordhalf_idx = 0;
#ifdef __LP64__
uint32_t cur_include_halfword = (uint32_t)cur_include_word;
#else
uint32_t cur_include_halfword = (uint16_t)cur_include_word;
#endif
while (1) {
if (cur_include_halfword) {
uintptr_t orig_quaterarr_word = main_quaterarr[subset_mask_widx * 2 + wordhalf_idx];
do {
uint32_t rqa_idx_lowbits = __builtin_ctz(cur_include_halfword);
cur_output_word |= ((orig_quaterarr_word >> (rqa_idx_lowbits * 2)) & 3) << (word_write_halfshift * 2);
if (++word_write_halfshift == BITCT2) {
*main_quaterarr_writer++ = cur_output_word;
word_write_halfshift = 0;
cur_output_word = 0;
}
cur_include_halfword &= cur_include_halfword - 1;
} while (cur_include_halfword);
}
if (wordhalf_idx) {
break;
}
wordhalf_idx++;
#ifdef __LP64__
cur_include_halfword = cur_include_word >> 32;
#else
cur_include_halfword = cur_include_word >> 16;
#endif
}
if (main_quaterarr_writer == main_quaterarr_write_last) {
if (word_write_halfshift == word_write_halfshift_end) {
if (word_write_halfshift_end) {
*main_quaterarr_writer = cur_output_word;
}
return;
}
}
}
subset_mask_widx++;
}
}
// blocked copy
while (1) {
const uintptr_t cur_include_word = *subset_mask++;
uint32_t wordhalf_idx = 0;
#ifdef __LP64__
uintptr_t cur_include_halfword = (uint32_t)cur_include_word;
#else
uint32_t cur_include_halfword = (uint16_t)cur_include_word;
#endif
while (1) {
uintptr_t orig_quaterarr_word = *main_quaterarr++;
while (cur_include_halfword) {
uint32_t rqa_idx_lowbits = CTZLU(cur_include_halfword);
uintptr_t halfword_invshifted = (~cur_include_halfword) >> rqa_idx_lowbits;
uintptr_t orig_quaterarr_curblock_unmasked = orig_quaterarr_word >> (rqa_idx_lowbits * 2);
uint32_t rqa_block_len = CTZLU(halfword_invshifted);
uint32_t block_len_limit = BITCT2 - word_write_halfshift;
cur_output_word |= orig_quaterarr_curblock_unmasked << (2 * word_write_halfshift);
if (rqa_block_len < block_len_limit) {
word_write_halfshift += rqa_block_len;
cur_output_word &= (ONELU << (word_write_halfshift * 2)) - ONELU;
} else {
// no need to mask, extra bits vanish off the high end
*main_quaterarr_writer++ = cur_output_word;
word_write_halfshift = rqa_block_len - block_len_limit;
if (word_write_halfshift) {
cur_output_word = (orig_quaterarr_curblock_unmasked >> (2 * block_len_limit)) & ((ONELU << (2 * word_write_halfshift)) - ONELU);
} else {
cur_output_word = 0;
}
}
cur_include_halfword &= (~(ONELU << (rqa_block_len + rqa_idx_lowbits))) + ONELU;
}
if (wordhalf_idx) {
break;
}
wordhalf_idx++;
#ifdef __LP64__
cur_include_halfword = cur_include_word >> 32;
#else
cur_include_halfword = cur_include_word >> 16;
#endif
}
if (main_quaterarr_writer == main_quaterarr_write_last) {
if (word_write_halfshift == word_write_halfshift_end) {
if (word_write_halfshift_end) {
*main_quaterarr_writer = cur_output_word;
}
return;
}
}
}
}
*/
uint32_t load_and_collapse_incl(uint32_t unfiltered_sample_ct, uint32_t sample_ct, const uintptr_t* __restrict sample_include, uintptr_t final_mask, uint32_t do_reverse, FILE* bedfile, uintptr_t* __restrict rawbuf, uintptr_t* __restrict mainbuf) {
assert(unfiltered_sample_ct);
uint32_t unfiltered_sample_ct4 = (unfiltered_sample_ct + 3) / 4;
if (unfiltered_sample_ct == sample_ct) {
rawbuf = mainbuf;
}
if (load_raw(unfiltered_sample_ct4, bedfile, rawbuf)) {
return RET_READ_FAIL;
}
if (unfiltered_sample_ct != sample_ct) {
copy_quaterarr_nonempty_subset(rawbuf, sample_include, unfiltered_sample_ct, sample_ct, mainbuf);
} else {
mainbuf[(unfiltered_sample_ct - 1) / BITCT2] &= final_mask;
}
if (do_reverse) {
reverse_loadbuf(sample_ct, (unsigned char*)mainbuf);
}
return 0;
}
/*
uint32_t load_and_collapse_incl_inplace(const uintptr_t* __restrict sample_include, uint32_t unfiltered_sample_ct, uint32_t sample_ct, uintptr_t final_mask, uint32_t do_reverse, FILE* bedfile, uintptr_t* __restrict mainbuf) {
// mainbuf must be large enough to store unfiltered data
uint32_t unfiltered_sample_ct4 = (unfiltered_sample_ct + 3) / 4;
if (load_raw(unfiltered_sample_ct4, bedfile, mainbuf)) {
return RET_READ_FAIL;
}
if (unfiltered_sample_ct == sample_ct) {
mainbuf[(unfiltered_sample_ct - 1) / BITCT2] &= final_mask;
} else {
inplace_quaterarr_proper_subset(sample_include, unfiltered_sample_ct, sample_ct, mainbuf);
}
if (do_reverse) {
reverse_loadbuf(sample_ct, (unsigned char*)mainbuf);
}
return 0;
}
*/
uint32_t load_and_split(uint32_t unfiltered_sample_ct, const uintptr_t* __restrict pheno_nm, const uintptr_t* __restrict pheno_c, FILE* bedfile, uintptr_t* __restrict rawbuf, uintptr_t* __restrict casebuf, uintptr_t* __restrict ctrlbuf) {
// add do_reverse later if needed
uintptr_t* rawbuf_end = &(rawbuf[unfiltered_sample_ct / BITCT2]);
uintptr_t case_word = 0;
uintptr_t ctrl_word = 0;
uint32_t unfiltered_sample_ct4 = (unfiltered_sample_ct + 3) / 4;
uint32_t case_shift2 = 0;
uint32_t ctrl_shift2 = 0;
uint32_t read_shift_max = BITCT2;
uint32_t sample_uidx = 0;
uint32_t read_shift;
uintptr_t read_word;
uintptr_t ulii;
if (load_raw(unfiltered_sample_ct4, bedfile, rawbuf)) {
return RET_READ_FAIL;
}
while (1) {
while (rawbuf < rawbuf_end) {
read_word = *rawbuf++;
for (read_shift = 0; read_shift < read_shift_max; sample_uidx++, read_shift++) {
if (is_set(pheno_nm, sample_uidx)) {
ulii = read_word & 3;
if (is_set(pheno_c, sample_uidx)) {
case_word |= ulii << case_shift2;
case_shift2 += 2;
if (case_shift2 == BITCT) {
*casebuf++ = case_word;
case_word = 0;
case_shift2 = 0;
}
} else {
ctrl_word |= ulii << ctrl_shift2;
ctrl_shift2 += 2;
if (ctrl_shift2 == BITCT) {
*ctrlbuf++ = ctrl_word;
ctrl_word = 0;
ctrl_shift2 = 0;
}
}
}
read_word >>= 2;
}
}
if (sample_uidx == unfiltered_sample_ct) {
if (case_shift2) {
*casebuf = case_word;
}
if (ctrl_shift2) {
*ctrlbuf = ctrl_word;
}
return 0;
}
rawbuf_end++;
read_shift_max = unfiltered_sample_ct % BITCT2;
}
}
void init_quaterarr_from_bitarr(const uintptr_t* __restrict bitarr, uintptr_t unfiltered_sample_ct, uintptr_t* __restrict new_quaterarr) {
// allows unfiltered_sample_ct == 0
uint32_t unfiltered_sample_ctl = BITCT_TO_WORDCT(unfiltered_sample_ct);
uintptr_t ulii;
uintptr_t uljj;
uintptr_t ulkk;
uintptr_t ulmm;
uint32_t bit_idx;
while (unfiltered_sample_ctl) {
ulii = ~(*bitarr++);
ulkk = FIVEMASK;
ulmm = FIVEMASK;
if (ulii) {
uljj = ulii >> BITCT2;
#ifdef __LP64__
ulii &= 0xffffffffLLU;
#else
ulii &= 0xffffLU;
#endif
if (ulii) {
do {
bit_idx = CTZLU(ulii);
ulkk &= ~(ONELU << (bit_idx * 2));
ulii &= ulii - 1;
} while (ulii);
}
if (uljj) {
do {
bit_idx = CTZLU(uljj);
ulmm &= ~(ONELU << (bit_idx * 2));
uljj &= uljj - 1;
} while (uljj);
}
}
*new_quaterarr++ = ulkk;
*new_quaterarr++ = ulmm;
--unfiltered_sample_ctl;
}
ulii = unfiltered_sample_ct & (BITCT - 1);
if (ulii) {
new_quaterarr--;
if (ulii < BITCT2) {
*new_quaterarr-- = 0;
} else {
ulii -= BITCT2;
}
*new_quaterarr &= (ONELU << (ulii * 2)) - ONELU;
}
}
void init_quaterarr_from_inverted_bitarr(const uintptr_t* __restrict inverted_bitarr, uintptr_t unfiltered_sample_ct, uintptr_t* __restrict new_quaterarr) {
// allows unfiltered_sample_ct == 0
uint32_t unfiltered_sample_ctl = BITCT_TO_WORDCT(unfiltered_sample_ct);
uintptr_t ulii;
uintptr_t uljj;
uintptr_t ulkk;
uintptr_t ulmm;
uint32_t bit_idx;
while (unfiltered_sample_ctl) {
ulii = *inverted_bitarr++;
ulkk = FIVEMASK;
ulmm = FIVEMASK;
if (ulii) {
uljj = ulii >> BITCT2;
#ifdef __LP64__
ulii &= 0xffffffffLLU;
#else
ulii &= 0xffffLU;
#endif
if (ulii) {
do {
bit_idx = CTZLU(ulii);
ulkk &= ~(ONELU << (bit_idx * 2));
ulii &= ulii - 1;
} while (ulii);
}
if (uljj) {
do {
bit_idx = CTZLU(uljj);
ulmm &= ~(ONELU << (bit_idx * 2));
uljj &= uljj - 1;
} while (uljj);
}
}
*new_quaterarr++ = ulkk;
*new_quaterarr++ = ulmm;
--unfiltered_sample_ctl;
}
ulii = unfiltered_sample_ct & (BITCT - 1);
if (ulii) {
new_quaterarr--;
if (ulii < BITCT2) {
*new_quaterarr-- = 0;
} else {
ulii -= BITCT2;
}
*new_quaterarr &= (ONELU << (ulii * 2)) - ONELU;
}
}
void quatervec_01_init_invert(const uintptr_t* __restrict source_quatervec, uintptr_t entry_ct, uintptr_t* __restrict target_quatervec) {
// Initializes a quatervec as the inverse of another.
// Some modifications needed for AVX2.
uint32_t vec_wsize = QUATERCT_TO_ALIGNED_WORDCT(entry_ct);
uint32_t rem = entry_ct & (BITCT - 1);
#ifdef __LP64__
const __m128i m1 = {FIVEMASK, FIVEMASK};
__m128i* tptr = (__m128i*)target_quatervec;
__m128i* sptr = (__m128i*)source_quatervec;
__m128i* tptr_end = (__m128i*)(&(target_quatervec[vec_wsize]));
uintptr_t* second_to_last;
while (tptr < tptr_end) {
*tptr++ = _mm_andnot_si128(*sptr++, m1);
}
if (rem) {
second_to_last = &(((uintptr_t*)tptr_end)[-2]);
if (rem > BITCT2) {
second_to_last[1] &= (~ZEROLU) >> ((BITCT - rem) * 2);
} else {
*second_to_last &= (~ZEROLU) >> ((BITCT2 - rem) * 2);
second_to_last[1] = 0;
}
}
#else
uintptr_t* tptr_end = &(target_quatervec[vec_wsize]);
while (target_quatervec < tptr_end) {
*target_quatervec++ = FIVEMASK & (~(*source_quatervec++));
}
if (rem) {
if (rem > BITCT2) {
target_quatervec[-1] &= (~ZEROLU) >> ((BITCT - rem) * 2);
} else {
target_quatervec[-2] &= (~ZEROLU) >> ((BITCT2 - rem) * 2);
target_quatervec[-1] = 0;
}
}
#endif
}
void bitvec_andnot_copy(const uintptr_t* __restrict source_vec, const uintptr_t* __restrict exclude_vec, uintptr_t word_ct, uintptr_t* __restrict target_vec) {
// target_vec := source_vec ANDNOT exclude_vec
// may write an extra word
assert(word_ct);
#ifdef __LP64__
__m128i* tptr = (__m128i*)target_vec;
__m128i* sptr = (__m128i*)source_vec;
__m128i* xptr = (__m128i*)exclude_vec;
__m128i* tptr_end = (__m128i*)(&(target_vec[round_up_pow2(word_ct, VEC_WORDS)]));
do {
*tptr++ = _mm_andnot_si128(*xptr++, *sptr++);
} while (tptr < tptr_end);
#else
uintptr_t* tptr_end = &(target_vec[word_ct]);
do {
*target_vec++ = (*source_vec++) & (~(*exclude_vec++));
} while (target_vec < tptr_end);
#endif
}
void apply_bitarr_mask_to_quaterarr_01(const uintptr_t* __restrict mask_bitarr, uintptr_t unfiltered_sample_ct, uintptr_t* main_quaterarr) {
// allows unfiltered_sample_ct == 0
uint32_t unfiltered_sample_ctl = BITCT_TO_WORDCT(unfiltered_sample_ct);
uintptr_t ulii;
uintptr_t uljj;
uintptr_t ulkk;
uintptr_t ulmm;
uint32_t bit_idx;
while (unfiltered_sample_ctl) {
ulii = ~(*mask_bitarr++);
ulkk = *main_quaterarr;
ulmm = main_quaterarr[1];
if (ulii) {
uljj = ulii >> BITCT2;
#ifdef __LP64__
ulii &= 0xffffffffLLU;
#else
ulii &= 0xffffLU;
#endif
if (ulii) {
do {
bit_idx = CTZLU(ulii);
ulkk &= ~(ONELU << (bit_idx * 2));
ulii &= ulii - 1;
} while (ulii);
}
if (uljj) {
do {
bit_idx = CTZLU(uljj);
ulmm &= ~(ONELU << (bit_idx * 2));
uljj &= uljj - 1;
} while (uljj);
}
}
*main_quaterarr++ = ulkk;
*main_quaterarr++ = ulmm;
--unfiltered_sample_ctl;
}
}
void apply_bitarr_excl_to_quaterarr_01(const uintptr_t* __restrict excl_bitarr, uintptr_t unfiltered_sample_ct, uintptr_t* __restrict main_quaterarr) {
assert(unfiltered_sample_ct);
uint32_t unfiltered_sample_ctl = BITCT_TO_WORDCT(unfiltered_sample_ct);
uintptr_t ulii;
uintptr_t uljj;
uintptr_t ulkk;
uintptr_t ulmm;
uint32_t bit_idx;
do {
ulii = *excl_bitarr++;
ulkk = *main_quaterarr;
ulmm = main_quaterarr[1];
if (ulii) {
uljj = ulii >> BITCT2;
#ifdef __LP64__
ulii &= 0xffffffffLLU;
#else
ulii &= 0xffffLU;
#endif
if (ulii) {
do {
bit_idx = CTZLU(ulii);
ulkk &= ~(ONELU << (bit_idx * 2));
ulii &= ulii - 1;
} while (ulii);
}
if (uljj) {
do {
bit_idx = CTZLU(uljj);
ulmm &= ~(ONELU << (bit_idx * 2));
uljj &= uljj - 1;
} while (uljj);
}
}
*main_quaterarr++ = ulkk;
*main_quaterarr++ = ulmm;
} while (--unfiltered_sample_ctl);
}
void apply_excl_intersect_to_quaterarr_01(const uintptr_t* __restrict excl_bitarr_1, const uintptr_t* __restrict excl_bitarr_2, uintptr_t unfiltered_sample_ct, uintptr_t* __restrict main_quaterarr) {
assert(unfiltered_sample_ct);
uint32_t unfiltered_sample_ctl = BITCT_TO_WORDCT(unfiltered_sample_ct);
uintptr_t ulii;
uintptr_t uljj;
uintptr_t ulkk;
uintptr_t ulmm;
uint32_t bit_idx;
do {
ulii = (*excl_bitarr_1++) & (*excl_bitarr_2++);
ulkk = *main_quaterarr;
ulmm = main_quaterarr[1];
if (ulii) {
uljj = ulii >> BITCT2;
#ifdef __LP64__
ulii &= 0xffffffffLLU;
#else
ulii &= 0xffffLU;
#endif
if (ulii) {
do {
bit_idx = CTZLU(ulii);
ulkk &= ~(ONELU << (bit_idx * 2));
ulii &= ulii - 1;
} while (ulii);
}
if (uljj) {
do {
bit_idx = CTZLU(uljj);
ulmm &= ~(ONELU << (bit_idx * 2));
uljj &= uljj - 1;
} while (uljj);
}
}
*main_quaterarr++ = ulkk;
*main_quaterarr++ = ulmm;
} while (--unfiltered_sample_ctl);
}
void quatervec_copy_only_01(const uintptr_t* __restrict input_quatervec, uintptr_t unfiltered_sample_ct, uintptr_t* __restrict output_quatervec) {
// initializes result_ptr bits 01 iff input_quatervec bits are 01
assert(unfiltered_sample_ct);
#ifdef __LP64__
const __m128i m1 = {FIVEMASK, FIVEMASK};
__m128i* vec2_read = (__m128i*)input_quatervec;
__m128i* read_end = &(vec2_read[QUATERCT_TO_VECCT(unfiltered_sample_ct)]);
__m128i* vec2_write = (__m128i*)output_quatervec;
__m128i loader;
do {
loader = *vec2_read++;
*vec2_write++ = _mm_and_si128(_mm_andnot_si128(_mm_srli_epi64(loader, 1), loader), m1);
} while (vec2_read < read_end);
#else
const uintptr_t* read_end = &(input_quatervec[QUATERCT_TO_ALIGNED_WORDCT(unfiltered_sample_ct)]);
uintptr_t loader;
do {
loader = *input_quatervec++;
*output_quatervec++ = loader & (~(loader >> 1)) & FIVEMASK;
} while (input_quatervec < read_end);
#endif
}
void quatervec_01_invert(uintptr_t unfiltered_sample_ct, uintptr_t* main_quatervec) {
uintptr_t* vec2_last = &(main_quatervec[unfiltered_sample_ct / BITCT2]);
uint32_t remainder = unfiltered_sample_ct & (BITCT2 - 1);
#ifdef __LP64__
const __m128i m1 = {FIVEMASK, FIVEMASK};
__m128i* vec2_128 = (__m128i*)main_quatervec;
__m128i* vec2_last128 = &(vec2_128[unfiltered_sample_ct / BITCT]);
while (vec2_128 < vec2_last128) {
*vec2_128 = _mm_xor_si128(*vec2_128, m1);
vec2_128++;
}
main_quatervec = (uintptr_t*)vec2_128;
if (main_quatervec != vec2_last) {
*main_quatervec = (*main_quatervec) ^ FIVEMASK;
main_quatervec++;
}
#else
while (main_quatervec != vec2_last) {
*main_quatervec = (*main_quatervec) ^ FIVEMASK;
main_quatervec++;
}
#endif
if (remainder) {
*vec2_last = *vec2_last ^ (FIVEMASK >> (2 * (BITCT2 - remainder)));
}
}
void vec_datamask(uintptr_t unfiltered_sample_ct, uint32_t matchval, uintptr_t* data_ptr, uintptr_t* mask_ptr, uintptr_t* result_ptr) {
// vec_ptr assumed to be standard 00/01 bit vector
// sets result_vec bits to 01 iff data_ptr bits are equal to matchval and
// vec_ptr bit is set, 00 otherwise.
// currently assumes matchval is not 1.
assert(unfiltered_sample_ct);
#ifdef __LP64__
__m128i* data_read = (__m128i*)data_ptr;
__m128i* mask_read = (__m128i*)mask_ptr;
__m128i* data_read_end = &(data_read[QUATERCT_TO_VECCT(unfiltered_sample_ct)]);
__m128i* writer = (__m128i*)result_ptr;
__m128i loader;
#else
uintptr_t* data_read_end = &(data_ptr[QUATERCT_TO_ALIGNED_WORDCT(unfiltered_sample_ct)]);
uintptr_t loader;
#endif
if (matchval) {
if (matchval == 2) {
#ifdef __LP64__
do {
loader = *data_read++;
*writer++ = _mm_and_si128(_mm_andnot_si128(loader, _mm_srli_epi64(loader, 1)), *mask_read++);
} while (data_read < data_read_end);
#else
do {
loader = *data_ptr++;
*result_ptr++ = (~loader) & (loader >> 1) & (*mask_ptr++);
} while (data_ptr < data_read_end);
#endif
} else {
#ifdef __LP64__
do {
loader = *data_read++;
*writer++ = _mm_and_si128(_mm_and_si128(loader, _mm_srli_epi64(loader, 1)), *mask_read++);
} while (data_read < data_read_end);
#else
do {
loader = *data_ptr++;
*result_ptr++ = loader & (loader >> 1) & (*mask_ptr++);
} while (data_ptr < data_read_end);
#endif
}
} else {
#ifdef __LP64__
do {
loader = *data_read++;
*writer++ = _mm_andnot_si128(_mm_or_si128(loader, _mm_srli_epi64(loader, 1)), *mask_read++);
} while (data_read < data_read_end);
#else
do {
loader = *data_ptr++;
*result_ptr++ = (~(loader | (loader >> 1))) & (*mask_ptr++);
} while (data_ptr < data_read_end);
#endif
}
}
/*
void vec_rotate_plink1_to_plink2(uintptr_t* lptr, uint32_t word_ct) {
#ifdef __LP64__
const __m128i m1 = {FIVEMASK, FIVEMASK};
__m128i* vptr = (__m128i*)lptr;
__m128i* vend = (__m128i*)(&(lptr[word_ct]));
__m128i vii;
__m128i vjj;
do {
// new high bit set iff old low bit was set
// new low bit set iff old bits differed
vii = *vptr;
vjj = _mm_and_si128(vii, m1); // old low bit
vii = _mm_and_si128(_mm_srli_epi64(vii, 1), m1); // old high bit, shifted
*vptr = _mm_or_si128(_mm_slli_epi64(vjj, 1), _mm_xor_si128(vii, vjj));
} while (++vptr != vend);
#else
uintptr_t* lend = &(lptr[word_ct]);
uintptr_t ulii;
uintptr_t uljj;
do {
ulii = *lptr;
uljj = ulii & FIVEMASK;
ulii = (ulii >> 1) & FIVEMASK;
*lptr = ulii ^ (uljj * 3);
} while (++lptr != lend);
#endif
}
*/
// this was "rotate_plink1_to_plink2_...", until I noticed that the plink2
// format should store alt allele counts instead of ref allele counts.
void rotate_plink1_to_a2ct_and_copy(uintptr_t* loadbuf, uintptr_t* writebuf, uintptr_t word_ct) {
// assumes positive word_ct
uintptr_t* loadbuf_end = &(loadbuf[word_ct]);
uintptr_t ulii;
uintptr_t uljj;
do {
ulii = *loadbuf++;
uljj = ulii & FIVEMASK;
ulii = (ulii >> 1) & FIVEMASK;
*writebuf++ = ulii ^ (uljj * 3);
} while (loadbuf < loadbuf_end);
}
void extract_collapsed_missing_bitfield(uintptr_t* lptr, uintptr_t unfiltered_sample_ct, uintptr_t* sample_include_quaterarr, uintptr_t sample_ct, uintptr_t* missing_bitfield) {
uint32_t word_ct = QUATERCT_TO_WORDCT(unfiltered_sample_ct);
uintptr_t sample_idx;
uintptr_t cur_word;
uintptr_t cur_mask;
uintptr_t cur_write;
uint32_t woffset;
uint32_t widx;
uint32_t uii;
if (unfiltered_sample_ct == sample_ct) {
cur_write = 0;
woffset = 0;
for (widx = 0; widx < word_ct; widx++) {
cur_word = *lptr++;
cur_word = cur_word & ((~cur_word) >> 1) & (*sample_include_quaterarr++);
while (cur_word) {
uii = CTZLU(cur_word) / 2;
cur_write |= ONELU << (woffset + uii);
cur_word &= cur_word - 1;
}
if (woffset) {
*missing_bitfield++ = cur_write;
cur_write = 0;
woffset = 0;
} else {
woffset = BITCT2;
}
}
if (woffset) {
*missing_bitfield++ = cur_write;
}
} else {
fill_ulong_zero(BITCT_TO_WORDCT(sample_ct), missing_bitfield);
sample_idx = 0;
for (widx = 0; sample_idx < sample_ct; widx++, lptr++) {
cur_mask = *sample_include_quaterarr++;
if (cur_mask) {
cur_word = *lptr;
cur_word = cur_word & ((~cur_word) >> 1) & cur_mask;
if (cur_mask == FIVEMASK) {
if (cur_word) {
uii = sample_idx;
do {
set_bit((CTZLU(cur_word) / 2) + uii, missing_bitfield);
cur_word &= cur_word - 1;
} while (cur_word);
}
sample_idx += BITCT2;
} else {
if (cur_word) {
do {
uii = CTZLU(cur_mask);
if ((cur_word >> uii) & 1) {
set_bit_ul(sample_idx, missing_bitfield);
}
sample_idx++;
cur_mask &= cur_mask - 1;
} while (cur_mask);
} else {
sample_idx += popcount2_long(cur_mask);
}
}
}
}
}
}
void hh_reset(unsigned char* loadbuf, uintptr_t* sample_include_quaterarr, uintptr_t unfiltered_sample_ct) {
uintptr_t sample_bidx = 0;
unsigned char* loadbuf_end = &(loadbuf[(unfiltered_sample_ct + 3) / 4]);
unsigned char* iicp;
unsigned char ucc;
unsigned char ucc2;
uintptr_t unfiltered_sample_ctd;
uint32_t* loadbuf_alias32;
uint32_t uii;
uint32_t ujj;
#ifdef __LP64__
uint32_t* sample_include_quaterarr_alias32;
__m128i* loadbuf_alias;
__m128i* iivp;
__m128i vii;
__m128i vjj;
if (!(((uintptr_t)loadbuf) & 15)) {
loadbuf_alias = (__m128i*)loadbuf;
iivp = (__m128i*)sample_include_quaterarr;
unfiltered_sample_ctd = unfiltered_sample_ct / 64;
for (; sample_bidx < unfiltered_sample_ctd; sample_bidx++) {
vii = *loadbuf_alias;
vjj = _mm_and_si128(_mm_andnot_si128(vii, _mm_srli_epi64(vii, 1)), *iivp++);
*loadbuf_alias++ = _mm_sub_epi64(vii, vjj);
}
loadbuf = (unsigned char*)loadbuf_alias;
iicp = (unsigned char*)iivp;
} else if (!(((uintptr_t)loadbuf) & 3)) {
loadbuf_alias32 = (uint32_t*)loadbuf;
sample_include_quaterarr_alias32 = (uint32_t*)sample_include_quaterarr;
unfiltered_sample_ctd = unfiltered_sample_ct / BITCT2;
for (; sample_bidx < unfiltered_sample_ctd; sample_bidx++) {
uii = *loadbuf_alias32;
ujj = ((uii >> 1) & (~uii)) & (*sample_include_quaterarr_alias32++);
*loadbuf_alias32++ = uii - ujj;
}
loadbuf = (unsigned char*)loadbuf_alias32;
iicp = (unsigned char*)sample_include_quaterarr_alias32;
} else {
iicp = (unsigned char*)sample_include_quaterarr;
}
#else
if (!(((uintptr_t)loadbuf) & 3)) {
loadbuf_alias32 = (uint32_t*)loadbuf;
unfiltered_sample_ctd = unfiltered_sample_ct / BITCT2;
for (; sample_bidx < unfiltered_sample_ctd; sample_bidx++) {
uii = *loadbuf_alias32;
ujj = ((uii >> 1) & (~uii)) & (*sample_include_quaterarr++);
*loadbuf_alias32++ = uii - ujj;
}
loadbuf = (unsigned char*)loadbuf_alias32;
}
iicp = (unsigned char*)sample_include_quaterarr;
#endif
for (; loadbuf < loadbuf_end;) {
ucc = *loadbuf;
ucc2 = ((ucc >> 1) & (~ucc)) & (*iicp++);
*loadbuf++ = ucc - ucc2;
}
}
void hh_reset_y(unsigned char* loadbuf, uintptr_t* sample_include_quaterarr, uintptr_t* sample_male_include_quaterarr, uintptr_t unfiltered_sample_ct) {
uintptr_t sample_bidx = 0;
unsigned char* loadbuf_end = &(loadbuf[(unfiltered_sample_ct + 3) / 4]);
unsigned char* iicp;
unsigned char* imicp;
unsigned char ucc;
unsigned char ucc2;
unsigned char ucc3;
uintptr_t unfiltered_sample_ctd;
uint32_t* loadbuf_alias32;
uint32_t uii;
uint32_t ujj;
uint32_t ukk;
#ifdef __LP64__
const __m128i m1 = {FIVEMASK, FIVEMASK};
uint32_t* sample_include_quaterarr_alias32;
uint32_t* sample_male_include_quaterarr_alias32;
__m128i* loadbuf_alias;
__m128i* iivp;
__m128i* imivp;
__m128i vii;
__m128i vjj;
__m128i vkk;
if (!(((uintptr_t)loadbuf) & 15)) {
loadbuf_alias = (__m128i*)loadbuf;
iivp = (__m128i*)sample_include_quaterarr;
imivp = (__m128i*)sample_male_include_quaterarr;
unfiltered_sample_ctd = unfiltered_sample_ct / 64;
for (; sample_bidx < unfiltered_sample_ctd; sample_bidx++) {
// sample_include_quaterarr & ~sample_male_include_quaterarr: force to 01
// sample_male_include_quaterarr: convert 10 to 01, keep everything else
vii = *imivp++;
vjj = *iivp++;
vkk = _mm_and_si128(*loadbuf_alias, _mm_or_si128(vii, _mm_slli_epi64(vii, 1)));
*loadbuf_alias++ = _mm_or_si128(_mm_andnot_si128(vii, vjj), _mm_sub_epi64(vkk, _mm_and_si128(_mm_andnot_si128(vkk, _mm_srli_epi64(vkk, 1)), m1)));
}
loadbuf = (unsigned char*)loadbuf_alias;
iicp = (unsigned char*)iivp;
imicp = (unsigned char*)imivp;
} else if (!(((uintptr_t)loadbuf) & 3)) {
loadbuf_alias32 = (uint32_t*)loadbuf;
sample_include_quaterarr_alias32 = (uint32_t*)sample_include_quaterarr;
sample_male_include_quaterarr_alias32 = (uint32_t*)sample_male_include_quaterarr;
unfiltered_sample_ctd = unfiltered_sample_ct / 16;
for (; sample_bidx < unfiltered_sample_ctd; sample_bidx++) {
uii = *sample_male_include_quaterarr_alias32++;
ujj = *sample_include_quaterarr_alias32++;
ukk = (*loadbuf_alias32) & (uii * 3);
*loadbuf_alias32++ = ((~uii) & ujj) | (ukk - ((~ukk) & (ukk >> 1) & 0x55555555));
}
loadbuf = (unsigned char*)loadbuf_alias32;
iicp = (unsigned char*)sample_include_quaterarr_alias32;
imicp = (unsigned char*)sample_male_include_quaterarr_alias32;
} else {
iicp = (unsigned char*)sample_include_quaterarr;
imicp = (unsigned char*)sample_male_include_quaterarr;
}
#else
if (!(((uintptr_t)loadbuf) & 3)) {
loadbuf_alias32 = (uint32_t*)loadbuf;
unfiltered_sample_ctd = unfiltered_sample_ct / 16;
for (; sample_bidx < unfiltered_sample_ctd; sample_bidx++) {
uii = *sample_male_include_quaterarr++;
ujj = *sample_include_quaterarr++;
ukk = (*loadbuf_alias32) & (uii * 3);
*loadbuf_alias32++ = ((~uii) & ujj) | (ukk - ((~ukk) & (ukk >> 1) & 0x55555555));
}
loadbuf = (unsigned char*)loadbuf_alias32;
}
iicp = (unsigned char*)sample_include_quaterarr;
imicp = (unsigned char*)sample_male_include_quaterarr;
#endif
for (; loadbuf < loadbuf_end;) {
ucc = *imicp++;
ucc2 = *iicp++;
ucc3 = (*loadbuf) & (ucc * 3);
*loadbuf++ = ((~ucc) & ucc2) | (ucc3 - ((~ucc3) & (ucc3 >> 1) & 0x55));
}
}
uint32_t alloc_raw_haploid_filters(uint32_t unfiltered_sample_ct, uint32_t hh_exists, uint32_t is_include, uintptr_t* sample_bitarr, uintptr_t* sex_male, uintptr_t** sample_raw_include_quatervec_ptr, uintptr_t** sample_raw_male_include_quatervec_ptr) {
uintptr_t unfiltered_sample_ctv2 = QUATERCT_TO_ALIGNED_WORDCT(unfiltered_sample_ct);
uintptr_t* sample_raw_male_include_quatervec;
if (hh_exists & (Y_FIX_NEEDED | NXMHH_EXISTS)) {
if (bigstack_alloc_ul(unfiltered_sample_ctv2, sample_raw_include_quatervec_ptr)) {
return 1;
}
if (is_include) {
init_quaterarr_from_bitarr(sample_bitarr, unfiltered_sample_ct, *sample_raw_include_quatervec_ptr);
} else {
init_quaterarr_from_inverted_bitarr(sample_bitarr, unfiltered_sample_ct, *sample_raw_include_quatervec_ptr);
}
}
if (hh_exists & (XMHH_EXISTS | Y_FIX_NEEDED)) {
if (bigstack_alloc_ul(unfiltered_sample_ctv2, sample_raw_male_include_quatervec_ptr)) {
return 1;
}
sample_raw_male_include_quatervec = *sample_raw_male_include_quatervec_ptr;
if (hh_exists & (Y_FIX_NEEDED | NXMHH_EXISTS)) {
memcpy(sample_raw_male_include_quatervec, *sample_raw_include_quatervec_ptr, unfiltered_sample_ctv2 * sizeof(intptr_t));
} else {
if (is_include) {
init_quaterarr_from_bitarr(sample_bitarr, unfiltered_sample_ct, sample_raw_male_include_quatervec);
} else {
init_quaterarr_from_inverted_bitarr(sample_bitarr, unfiltered_sample_ct, sample_raw_male_include_quatervec);
}
}
apply_bitarr_mask_to_quaterarr_01(sex_male, unfiltered_sample_ct, sample_raw_male_include_quatervec);
}
return 0;
}
void haploid_fix_multiple(uintptr_t* marker_exclude, uintptr_t marker_uidx_start, uintptr_t marker_ct, Chrom_info* chrom_info_ptr, uint32_t hh_exists, uint32_t set_hh_missing, uint32_t set_mixed_mt_missing, uintptr_t* sample_raw_include2, uintptr_t* sample_raw_male_include2, uintptr_t unfiltered_sample_ct, uintptr_t byte_ct_per_marker, unsigned char* loadbuf) {
uintptr_t marker_idx = 0;
uintptr_t marker_uidx = next_unset_ul_unsafe(marker_exclude, marker_uidx_start);
uint32_t chrom_fo_idx = get_variant_chrom_fo_idx(chrom_info_ptr, marker_uidx);
uint32_t chrom_idx;
uint32_t is_x;
uint32_t is_y;
uint32_t is_mt;
uint32_t is_haploid;
uintptr_t chrom_end;
uintptr_t marker_idx_chrom_end;
while (marker_idx < marker_ct) {
chrom_idx = chrom_info_ptr->chrom_file_order[chrom_fo_idx];
chrom_end = chrom_info_ptr->chrom_fo_vidx_start[chrom_fo_idx + 1];
is_x = (chrom_info_ptr->xymt_codes[X_OFFSET] == (int32_t)chrom_idx);
is_y = (chrom_info_ptr->xymt_codes[Y_OFFSET] == (int32_t)chrom_idx);
is_mt = (chrom_info_ptr->xymt_codes[MT_OFFSET] == (int32_t)chrom_idx);
is_haploid = IS_SET(chrom_info_ptr->haploid_mask, chrom_idx);
marker_idx_chrom_end = marker_idx + chrom_end - marker_uidx - popcount_bit_idx(marker_exclude, marker_uidx, chrom_end);
if (marker_idx_chrom_end > marker_ct) {
marker_idx_chrom_end = marker_ct;
}
if (is_haploid && set_hh_missing) {
if (is_x) {
if (hh_exists & XMHH_EXISTS) {
for (; marker_idx < marker_idx_chrom_end; marker_idx++) {
hh_reset(&(loadbuf[marker_idx * byte_ct_per_marker]), sample_raw_male_include2, unfiltered_sample_ct);
}
}
} else if (is_y) {
if (hh_exists & Y_FIX_NEEDED) {
for (; marker_idx < marker_idx_chrom_end; marker_idx++) {
hh_reset_y(&(loadbuf[marker_idx * byte_ct_per_marker]), sample_raw_include2, sample_raw_male_include2, unfiltered_sample_ct);
}
}
} else if (hh_exists & NXMHH_EXISTS) {
for (; marker_idx < marker_idx_chrom_end; marker_idx++) {
hh_reset(&(loadbuf[marker_idx * byte_ct_per_marker]), sample_raw_include2, unfiltered_sample_ct);
}
}
} else if (is_mt && set_mixed_mt_missing) {
for (; marker_idx < marker_idx_chrom_end; marker_idx++) {
hh_reset(&(loadbuf[marker_idx * byte_ct_per_marker]), sample_raw_include2, unfiltered_sample_ct);
}
}
marker_idx = marker_idx_chrom_end;
chrom_fo_idx++;
}
}
void force_missing(unsigned char* loadbuf, uintptr_t* force_missing_include2, uintptr_t unfiltered_sample_ct) {
uintptr_t sample_bidx = 0;
unsigned char* loadbuf_end = &(loadbuf[(unfiltered_sample_ct + 3) / 4]);
unsigned char* fmicp;
unsigned char ucc;
unsigned char ucc2;
uintptr_t unfiltered_sample_ctd;
uint32_t* loadbuf_alias32;
uint32_t uii;
uint32_t ujj;
#ifdef __LP64__
uint32_t* force_missing_include2_alias32;
__m128i* loadbuf_alias;
__m128i* fmivp;
__m128i vii;
__m128i vjj;
if (!(((uintptr_t)loadbuf) & 15)) {
loadbuf_alias = (__m128i*)loadbuf;
fmivp = (__m128i*)force_missing_include2;
unfiltered_sample_ctd = unfiltered_sample_ct / 64;
for (; sample_bidx < unfiltered_sample_ctd; sample_bidx++) {
vii = *loadbuf_alias;
vjj = *fmivp++;
vii = _mm_or_si128(vii, vjj);
vjj = _mm_slli_epi64(vjj, 1);
*loadbuf_alias++ = _mm_andnot_si128(vjj, vii);
}
loadbuf = (unsigned char*)loadbuf_alias;
fmicp = (unsigned char*)fmivp;
} else if (!(((uintptr_t)loadbuf) & 3)) {
loadbuf_alias32 = (uint32_t*)loadbuf;
force_missing_include2_alias32 = (uint32_t*)force_missing_include2;
unfiltered_sample_ctd = unfiltered_sample_ct / BITCT2;
for (; sample_bidx < unfiltered_sample_ctd; sample_bidx++) {
uii = *loadbuf_alias32;
ujj = *force_missing_include2_alias32++;
uii |= ujj;
ujj <<= 1;
*loadbuf_alias32++ = uii & (~ujj);
}
loadbuf = (unsigned char*)loadbuf_alias32;
fmicp = (unsigned char*)force_missing_include2_alias32;
} else {
fmicp = (unsigned char*)force_missing_include2;
}
#else
if (!(((uintptr_t)loadbuf) & 3)) {
loadbuf_alias32 = (uint32_t*)loadbuf;
unfiltered_sample_ctd = unfiltered_sample_ct / BITCT2;
for (; sample_bidx < unfiltered_sample_ctd; sample_bidx++) {
uii = *loadbuf_alias32;
ujj = *force_missing_include2++;
uii |= ujj;
ujj <<= 1;
*loadbuf_alias32++ = uii & (~ujj);
}
loadbuf = (unsigned char*)loadbuf_alias32;
}
fmicp = (unsigned char*)force_missing_include2;
#endif
for (; loadbuf < loadbuf_end;) {
ucc = *loadbuf;
ucc2 = *fmicp++;
ucc |= ucc2;
ucc2 <<= 1;
*loadbuf++ = ucc & (~ucc2);
}
}
int32_t open_and_size_string_list(char* fname, FILE** infile_ptr, uintptr_t* list_len_ptr, uintptr_t* max_str_len_ptr) {
// assumes file is not open yet, and g_textbuf is safe to clobber
uint32_t max_len = 0;
uintptr_t line_idx = 0;
uintptr_t list_len = 0;
int32_t retval = 0;
char* bufptr;
uint32_t cur_len;
if (fopen_checked(fname, "r", infile_ptr)) {
goto open_and_size_string_list_ret_OPEN_FAIL;
}
g_textbuf[MAXLINELEN - 1] = ' ';
while (fgets(g_textbuf, MAXLINELEN, *infile_ptr)) {
line_idx++;
if (!g_textbuf[MAXLINELEN - 1]) {
LOGERRPRINTFWW("Error: Line %" PRIuPTR " of %s is pathologically long.\n", line_idx, fname);
goto open_and_size_string_list_ret_INVALID_FORMAT;
}
bufptr = skip_initial_spaces(g_textbuf);
if (is_eoln_kns(*bufptr)) {
continue;
}
// don't complain about more than one entry on a line for now
list_len++;
cur_len = strlen_se(bufptr);
if (cur_len >= max_len) {
max_len = cur_len + 1;
}
}
if (!feof(*infile_ptr)) {
goto open_and_size_string_list_ret_READ_FAIL;
}
*list_len_ptr = list_len;
*max_str_len_ptr = max_len;
while (0) {
open_and_size_string_list_ret_OPEN_FAIL:
retval = RET_OPEN_FAIL;
break;
open_and_size_string_list_ret_READ_FAIL:
retval = RET_READ_FAIL;
break;
open_and_size_string_list_ret_INVALID_FORMAT:
retval = RET_INVALID_FORMAT;
break;
}
return retval;
}
int32_t load_string_list(FILE** infile_ptr, uintptr_t max_str_len, char* str_list) {
// assumes file is open (probably by open_and_size_string_list), and
// g_textbuf is safe to clobber
int32_t retval = 0;
char* bufptr;
uint32_t cur_len;
rewind(*infile_ptr);
while (fgets(g_textbuf, MAXLINELEN, *infile_ptr)) {
bufptr = skip_initial_spaces(g_textbuf);
if (is_eoln_kns(*bufptr)) {
continue;
}
cur_len = strlen_se(bufptr);
memcpy(str_list, bufptr, cur_len);
str_list[cur_len] = '\0';
str_list = &(str_list[max_str_len]);
}
if (!feof(*infile_ptr)) {
goto load_string_list_ret_READ_FAIL;
}
while (0) {
load_string_list_ret_READ_FAIL:
retval = RET_READ_FAIL;
break;
}
return retval;
}
int32_t open_and_skip_first_lines(FILE** infile_ptr, char* fname, char* loadbuf, uintptr_t loadbuf_size, uint32_t lines_to_skip) {
uint32_t line_idx;
loadbuf[loadbuf_size - 1] = ' ';
if (fopen_checked(fname, "r", infile_ptr)) {
return RET_OPEN_FAIL;
}
for (line_idx = 1; line_idx <= lines_to_skip; line_idx++) {
if (!fgets(loadbuf, loadbuf_size, *infile_ptr)) {
if (feof(*infile_ptr)) {
LOGERRPRINTFWW("Error: Fewer lines than expected in %s.\n", fname);
return RET_INVALID_FORMAT;
} else {
return RET_READ_FAIL;
}
}
if (!(loadbuf[loadbuf_size - 1])) {
if ((loadbuf_size == MAXLINELEN) || (loadbuf_size == MAXLINEBUFLEN)) {
LOGERRPRINTFWW("Error: Line %u of %s is pathologically long.\n", line_idx, fname);
return RET_INVALID_FORMAT;
} else {
return RET_NOMEM;
}
}
}
return 0;
}
int32_t load_to_first_token(FILE* infile, uintptr_t loadbuf_size, char comment_char, const char* file_descrip, char* loadbuf, char** bufptr_ptr, uintptr_t* line_idx_ptr) {
uintptr_t line_idx = 0;
while (fgets(loadbuf, loadbuf_size, infile)) {
line_idx++;
if (!(loadbuf[loadbuf_size - 1])) {
// PLINK 1.9 has two text line loading modes: "regular" and "long".
// * "Regular" mode limits lines to about MAXLINELEN (about 128k as of
// this writing) characters.
// * "Long" mode theoretically accepts lines up to about MAXLINEBUFLEN
// (~2 GB) characters but degrades gracefully if less memory is
// available (in that case, an out-of-memory instead of an
// invalid-format error is reported on fgets overflow). Any long
// buffer size larger than MAXLINELEN should work properly with
// plink_common.
if ((loadbuf_size == MAXLINELEN) || (loadbuf_size == MAXLINEBUFLEN)) {
LOGERRPRINTF("Error: Line %" PRIuPTR " of %s is pathologically long.\n", line_idx, file_descrip);
return RET_INVALID_FORMAT;
} else {
return RET_NOMEM;
}
}
*bufptr_ptr = skip_initial_spaces(loadbuf);
if (!is_eoln_kns(**bufptr_ptr)) {
if ((**bufptr_ptr) != comment_char) {
*line_idx_ptr = line_idx;
return 0;
}
}
}
if (!feof(infile)) {
return RET_READ_FAIL;
}
LOGERRPRINTF("Error: Empty %s.\n", file_descrip);
return RET_INVALID_FORMAT;
}
int32_t open_and_load_to_first_token(FILE** infile_ptr, char* fname, uintptr_t loadbuf_size, char comment_char, const char* file_descrip, char* loadbuf, char** bufptr_ptr, uintptr_t* line_idx_ptr) {
loadbuf[loadbuf_size - 1] = ' ';
if (fopen_checked(fname, "r", infile_ptr)) {
return RET_OPEN_FAIL;
}
return load_to_first_token(*infile_ptr, loadbuf_size, comment_char, file_descrip, loadbuf, bufptr_ptr, line_idx_ptr);
}
int32_t scan_max_strlen(char* fname, uint32_t colnum, uint32_t colnum2, uint32_t headerskip, char skipchar, uintptr_t* max_str_len_ptr, uintptr_t* max_str2_len_ptr) {
// colnum and colnum2 are 1-based indices. If colnum2 is zero, only colnum
// is scanned.
// Includes terminating null in lengths.
FILE* infile = nullptr;
uintptr_t loadbuf_size = bigstack_left();
uintptr_t max_str_len = *max_str_len_ptr;
uintptr_t max_str2_len = 0;
char* loadbuf = (char*)g_bigstack_base;
uint32_t colmin;
uint32_t coldiff;
char* str1_ptr;
char* str2_ptr;
char cc;
uintptr_t cur_str_len;
uintptr_t line_idx;
int32_t retval;
if (loadbuf_size > MAXLINEBUFLEN) {
loadbuf_size = MAXLINEBUFLEN;
} else if (loadbuf_size <= MAXLINELEN) {
goto scan_max_strlen_ret_NOMEM;
}
retval = open_and_skip_first_lines(&infile, fname, loadbuf, loadbuf_size, headerskip);
if (retval) {
goto scan_max_strlen_ret_1;
}
if (colnum < colnum2) {
max_str2_len = *max_str2_len_ptr;
colmin = colnum - 1;
coldiff = colnum2 - colnum;
} else if (colnum2) {
max_str2_len = max_str_len;
max_str_len = *max_str2_len_ptr;
colmin = colnum2 - 1;
coldiff = colnum - colnum2;
} else {
colmin = colnum - 1;
coldiff = 0;
colnum2 = 0xffffffffU;
}
line_idx = headerskip;
while (fgets(loadbuf, loadbuf_size, infile)) {
line_idx++;
if (!(loadbuf[loadbuf_size - 1])) {
if (loadbuf_size == MAXLINEBUFLEN) {
LOGPREPRINTFWW("Error: Line %" PRIuPTR " of %s is pathologically long.\n", line_idx, fname);
goto scan_max_strlen_ret_INVALID_FORMAT_2;
} else {
goto scan_max_strlen_ret_NOMEM;
}
}
str1_ptr = skip_initial_spaces(loadbuf);
cc = *str1_ptr;
if (is_eoln_kns(cc) || (cc == skipchar)) {
continue;
}
str1_ptr = next_token_multz(str1_ptr, colmin);
str2_ptr = next_token_multz(str1_ptr, coldiff);
if (no_more_tokens_kns(str2_ptr)) {
// probably want option for letting this slide in the future
LOGPREPRINTFWW("Error: Line %" PRIuPTR " of %s has fewer tokens than expected.\n", line_idx, fname);
goto scan_max_strlen_ret_INVALID_FORMAT_2;
}
cur_str_len = strlen_se(str1_ptr);
if (cur_str_len >= max_str_len) {
max_str_len = cur_str_len + 1;
}
if (coldiff) {
cur_str_len = strlen_se(str2_ptr);
if (cur_str_len >= max_str2_len) {
max_str2_len = cur_str_len + 1;
}
}
}
if (!feof(infile)) {
goto scan_max_strlen_ret_READ_FAIL;
}
if (colnum < colnum2) {
*max_str_len_ptr = max_str_len;
if (coldiff) {
*max_str2_len_ptr = max_str2_len;
}
} else {
*max_str_len_ptr = max_str2_len;
*max_str2_len_ptr = max_str_len;
}
while (0) {
scan_max_strlen_ret_NOMEM:
retval = RET_NOMEM;
break;
scan_max_strlen_ret_READ_FAIL:
retval = RET_READ_FAIL;
break;
scan_max_strlen_ret_INVALID_FORMAT_2:
logerrprintb();
retval = RET_INVALID_FORMAT;
break;
}
scan_max_strlen_ret_1:
fclose_cond(infile);
return retval;
}
int32_t scan_max_fam_indiv_strlen(char* fname, uint32_t colnum, uintptr_t* max_sample_id_len_ptr) {
// colnum is a 1-based index with the FID column number; IID column is
// assumed to follow.
// Includes terminating null in lengths.
FILE* infile = nullptr;
uintptr_t loadbuf_size = bigstack_left();
uintptr_t max_sample_id_len = *max_sample_id_len_ptr;
uintptr_t line_idx = 0;
char* loadbuf = (char*)g_bigstack_base;
char* bufptr;
char* bufptr2;
uintptr_t cur_sample_id_len;
int32_t retval;
colnum--;
if (loadbuf_size > MAXLINEBUFLEN) {
loadbuf_size = MAXLINEBUFLEN;
} else if (loadbuf_size <= MAXLINELEN) {
goto scan_max_fam_indiv_strlen_ret_NOMEM;
}
retval = open_and_skip_first_lines(&infile, fname, loadbuf, loadbuf_size, 0);
if (retval) {
goto scan_max_fam_indiv_strlen_ret_1;
}
while (fgets(loadbuf, loadbuf_size, infile)) {
line_idx++;
if (!(loadbuf[loadbuf_size - 1])) {
if (loadbuf_size == MAXLINEBUFLEN) {
LOGPREPRINTFWW("Error: Line %" PRIuPTR " of %s is pathologically long.\n", line_idx, fname);
goto scan_max_fam_indiv_strlen_ret_INVALID_FORMAT_2;
} else {
goto scan_max_fam_indiv_strlen_ret_NOMEM;
}
}
bufptr = skip_initial_spaces(loadbuf);
if (is_eoln_kns(*bufptr)) {
continue;
}
bufptr = next_token_multz(bufptr, colnum);
bufptr2 = next_token(bufptr);
if (no_more_tokens_kns(bufptr2)) {
LOGPREPRINTFWW("Error: Line %" PRIuPTR " of %s has fewer tokens than expected.\n", line_idx, fname);
goto scan_max_fam_indiv_strlen_ret_INVALID_FORMAT_2;
}
cur_sample_id_len = strlen_se(bufptr) + strlen_se(bufptr2) + 2;
if (cur_sample_id_len > max_sample_id_len) {
max_sample_id_len = cur_sample_id_len;
}
}
if (!feof(infile)) {
goto scan_max_fam_indiv_strlen_ret_READ_FAIL;
}
*max_sample_id_len_ptr = max_sample_id_len;
while (0) {
scan_max_fam_indiv_strlen_ret_NOMEM:
retval = RET_NOMEM;
break;
scan_max_fam_indiv_strlen_ret_READ_FAIL:
retval = RET_READ_FAIL;
break;
scan_max_fam_indiv_strlen_ret_INVALID_FORMAT_2:
logerrprintb();
retval = RET_INVALID_FORMAT;
break;
}
scan_max_fam_indiv_strlen_ret_1:
fclose_cond(infile);
return retval;
}
/*
void inplace_collapse_uint32(uint32_t* item_arr, uint32_t unfiltered_ct, uintptr_t* exclude_arr, uint32_t filtered_ct) {
if (unfiltered_ct == filtered_ct) {
return;
}
uint32_t item_uidx = next_set_unsafe(exclude_arr, 0);
uint32_t item_idx = item_uidx;
for (; item_idx < filtered_ct; item_idx++, item_uidx++) {
next_unset_unsafe_ck(exclude_arr, &item_uidx);
item_arr[item_idx] = item_arr[item_uidx];
}
}
*/
void inplace_collapse_uint32_incl(uint32_t* item_arr, uint32_t unfiltered_ct, uintptr_t* incl_arr, uint32_t filtered_ct) {
if (unfiltered_ct == filtered_ct) {
return;
}
uint32_t item_uidx = next_unset_unsafe(incl_arr, 0);
uint32_t item_idx = item_uidx;
for (; item_idx < filtered_ct; item_idx++, item_uidx++) {
next_set_unsafe_ck(incl_arr, &item_uidx);
item_arr[item_idx] = item_arr[item_uidx];
}
}
char* alloc_and_init_collapsed_arr(char* item_arr, uintptr_t item_len, uintptr_t unfiltered_ct, uintptr_t* exclude_arr, uintptr_t filtered_ct, uint32_t read_only) {
uint32_t item_uidx = 0;
char* new_arr;
char* wptr;
char* wptr_end;
uintptr_t item_uidx_stop;
uintptr_t delta;
if (read_only && (unfiltered_ct == filtered_ct)) {
return item_arr;
}
if (bigstack_alloc_c(filtered_ct * item_len, &new_arr)) {
return nullptr;
}
wptr = new_arr;
wptr_end = &(new_arr[filtered_ct * item_len]);
while (wptr < wptr_end) {
item_uidx = next_unset_ul_unsafe(exclude_arr, item_uidx);
item_uidx_stop = next_set_ul(exclude_arr, item_uidx, unfiltered_ct);
delta = item_uidx_stop - item_uidx;
memcpy(wptr, &(item_arr[item_uidx * item_len]), delta * item_len);
wptr = &(wptr[delta * item_len]);
item_uidx = item_uidx_stop;
}
return new_arr;
}
char* alloc_and_init_collapsed_arr_incl(char* item_arr, uintptr_t item_len, uintptr_t unfiltered_ct, uintptr_t* include_arr, uintptr_t filtered_ct, uint32_t read_only) {
uint32_t item_uidx = 0;
char* new_arr;
char* wptr;
char* wptr_end;
uintptr_t item_uidx_stop;
uintptr_t delta;
if (read_only && (unfiltered_ct == filtered_ct)) {
return item_arr;
}
if (bigstack_alloc_c(filtered_ct * item_len, &new_arr)) {
return nullptr;
}
wptr = new_arr;
wptr_end = &(new_arr[filtered_ct * item_len]);
do {
item_uidx = next_set_ul_unsafe(include_arr, item_uidx);
item_uidx_stop = next_unset_ul(include_arr, item_uidx, unfiltered_ct);
delta = item_uidx_stop - item_uidx;
memcpy(wptr, &(item_arr[item_uidx * item_len]), delta * item_len);
wptr = &(wptr[delta * item_len]);
item_uidx = item_uidx_stop;
} while (wptr < wptr_end);
return new_arr;
}
void inplace_delta_collapse_arr(char* item_arr, uintptr_t item_len, uintptr_t filtered_ct_orig, uintptr_t filtered_ct_new, uintptr_t* exclude_orig, uintptr_t* exclude_new) {
// if this sort of collapse function is ever in an important loop, check
// whether specialized 4-byte and 8-byte versions are much faster
uintptr_t* exclude_orig_start = exclude_orig;
char* write_end = &(item_arr[filtered_ct_new * item_len]);
uintptr_t read_idx = 1;
uint32_t uii = 0;
char* write_ptr;
uintptr_t ulii;
uintptr_t uljj;
uint32_t read_uidx;
uint32_t ujj;
if (filtered_ct_new == filtered_ct_orig) {
return;
}
// find location of first newly excluded item
while (1) {
ulii = *exclude_orig;
uljj = *exclude_new;
if (ulii != uljj) {
break;
}
uii += popcount_long(ulii);
exclude_orig++;
exclude_new++;
}
exclude_new -= ((uintptr_t)(exclude_orig - exclude_orig_start));
read_uidx = BITCT * ((uintptr_t)(exclude_orig - exclude_orig_start));
ujj = CTZLU(ulii ^ uljj);
read_uidx += ujj;
uii += popcount_long(ulii & ((ONELU << ujj) - ONELU));
uii = read_uidx - uii; // now equal to # initial filtered indices skipped
filtered_ct_new -= uii;
item_arr = &(item_arr[uii * item_len]);
write_ptr = item_arr;
read_uidx++;
for (; write_ptr < write_end; read_uidx++, read_idx++) {
next_unset_unsafe_ck(exclude_orig_start, &read_uidx);
if (IS_SET(exclude_new, read_uidx)) {
continue;
}
memcpy(write_ptr, &(item_arr[read_idx * item_len]), item_len);
write_ptr = &(write_ptr[item_len]);
}
}
void inplace_delta_collapse_bitfield(uintptr_t* read_ptr, uint32_t filtered_ct_new, uintptr_t* exclude_orig, uintptr_t* exclude_new) {
// only guaranteed to zero out trailing bits up to the nearest 16-byte
// boundary on 64-bit systems
uintptr_t* write_ptr = read_ptr;
uintptr_t readw = *read_ptr++;
uintptr_t writew = 0;
uint32_t item_uidx = 0;
uint32_t item_mwidx = 0;
uint32_t item_idx = 0;
for (; item_idx < filtered_ct_new; item_uidx++) {
next_unset_unsafe_ck(exclude_orig, &item_uidx);
if (!is_set(exclude_new, item_uidx)) {
if ((readw >> item_mwidx) & 1) {
writew |= ONELU << (item_idx % BITCT);
}
if (!((++item_idx) % BITCT)) {
*write_ptr++ = writew;
writew = 0;
}
}
if (++item_mwidx == BITCT) {
item_mwidx = 0;
readw = *read_ptr++;
}
}
if (write_ptr < read_ptr) {
*write_ptr++ = writew;
if (write_ptr < read_ptr) {
*write_ptr = 0;
}
}
}
void copy_bitarr_subset_excl(const uintptr_t* __restrict raw_bitarr, const uintptr_t* __restrict subset_excl, uint32_t raw_bitarr_size, uint32_t subset_size, uintptr_t* __restrict output_bitarr) {
uintptr_t cur_write = 0;
uint32_t item_uidx = 0;
uint32_t write_bit = 0;
uint32_t item_idx = 0;
uint32_t item_uidx_stop;
if (!subset_excl[0]) {
item_uidx = next_set(subset_excl, 0, raw_bitarr_size & (~(BITCT - 1))) & (~(BITCT - 1));
memcpy(output_bitarr, raw_bitarr, item_uidx / 8);
item_idx = item_uidx;
output_bitarr = &(output_bitarr[item_uidx / BITCT]);
}
while (item_idx < subset_size) {
item_uidx = next_unset_unsafe(subset_excl, item_uidx);
item_uidx_stop = next_set(subset_excl, item_uidx, raw_bitarr_size);
item_idx += item_uidx_stop - item_uidx;
do {
cur_write |= ((raw_bitarr[item_uidx / BITCT] >> (item_uidx % BITCT)) & 1) << write_bit;
if (++write_bit == BITCT) {
*output_bitarr++ = cur_write;
cur_write = 0;
write_bit = 0;
}
} while (++item_uidx < item_uidx_stop);
}
if (write_bit) {
*output_bitarr = cur_write;
}
}
void copy_bitarr_subset(const uintptr_t* __restrict raw_bitarr, const uintptr_t* __restrict subset_mask, uint32_t raw_bitarr_size, uint32_t subset_size, uintptr_t* __restrict output_bitarr) {
// full-blown blocked copy not worth it due to undefined CTZLU(0), >> 64,
// << 64
uintptr_t cur_output_word = 0;
uint32_t item_uidx = 0;
uint32_t word_write_shift = 0;
uint32_t item_idx = 0;
uint32_t item_uidx_stop;
if (!(~subset_mask[0])) {
item_uidx = next_unset(subset_mask, 0, raw_bitarr_size & (~(BITCT - 1))) & (~(BITCT - 1));
memcpy(output_bitarr, raw_bitarr, item_uidx / 8);
item_idx = item_uidx;
output_bitarr = &(output_bitarr[item_uidx / BITCT]);
}
while (item_idx < subset_size) {
item_uidx = next_set_unsafe(subset_mask, item_uidx);
// can speed this up a bit once we have a guaranteed unset bit at the end
item_uidx_stop = next_unset(subset_mask, item_uidx, raw_bitarr_size);
item_idx += item_uidx_stop - item_uidx;
do {
cur_output_word |= ((raw_bitarr[item_uidx / BITCT] >> (item_uidx % BITCT)) & 1) << word_write_shift;
if (++word_write_shift == BITCT) {
*output_bitarr++ = cur_output_word;
cur_output_word = 0;
word_write_shift = 0;
}
} while (++item_uidx < item_uidx_stop);
}
if (word_write_shift) {
*output_bitarr = cur_output_word;
}
}
void uncollapse_copy_flip_include_arr(uintptr_t* collapsed_include_arr, uintptr_t unfiltered_ct, uintptr_t* exclude_arr, uintptr_t* output_exclude_arr) {
uintptr_t unfiltered_ctl = BITCT_TO_WORDCT(unfiltered_ct);
uintptr_t* output_exclude_true_end = &(output_exclude_arr[unfiltered_ctl]);
uintptr_t* output_exclude_end = &(output_exclude_arr[unfiltered_ct / BITCT]);
uintptr_t cea_read = 0;
uint32_t read_bit = BITCT;
uint32_t write_bit;
uintptr_t cur_write;
uintptr_t cur_read = 0;
if (!exclude_arr[0]) {
// copy-with-possible-offset is substantially slower, so treat initial lack
// of offset as a special case
for (cur_read = 0; cur_read < unfiltered_ctl; cur_read++) {
*output_exclude_arr++ = ~(*collapsed_include_arr++);
if (*(++exclude_arr)) {
break;
}
}
}
while (output_exclude_arr < output_exclude_end) {
cur_write = *exclude_arr++;
// want efficient handling of all-zeroes and all-ones here
if (cur_write) {
cur_read = ~cur_write;
uncollapse_copy_flip_include_arr_loop:
while (cur_read) {
write_bit = CTZLU(cur_read);
if (read_bit == BITCT) {
cea_read = ~(*collapsed_include_arr++);
read_bit = 0;
}
cur_write |= (cea_read & ONELU) << write_bit;
cea_read >>= 1;
read_bit++;
cur_read &= cur_read - ONELU;
}
*output_exclude_arr = cur_write;
} else {
if (read_bit == BITCT) {
*output_exclude_arr = ~(*collapsed_include_arr++);
} else {
cur_write = cea_read;
cea_read = ~(*collapsed_include_arr++);
*output_exclude_arr = cur_write | (cea_read << (BITCT - read_bit));
cea_read >>= read_bit;
}
}
output_exclude_arr++;
}
if (output_exclude_arr < output_exclude_true_end) {
cur_write = *exclude_arr++;
cur_read = (~cur_write) & ((ONELU << (unfiltered_ct % BITCT)) - ONELU);
goto uncollapse_copy_flip_include_arr_loop;
}
}
void copy_when_nonmissing(uintptr_t* loadbuf, char* source, uintptr_t elem_size, uintptr_t unfiltered_sample_ct, uintptr_t missing_ct, char* dest) {
uintptr_t* loadbuf_end = &(loadbuf[QUATERCT_TO_WORDCT(unfiltered_sample_ct)]);
uintptr_t last_missing_p1 = 0;
uintptr_t sample_idx_offset = 0;
uintptr_t cur_word;
uintptr_t new_missing_idx;
uintptr_t diff;
if (!missing_ct) {
memcpy(dest, source, unfiltered_sample_ct * elem_size);
return;
}
do {
cur_word = *loadbuf++;
cur_word = cur_word & (~(cur_word >> 1)) & FIVEMASK;
while (cur_word) {
new_missing_idx = sample_idx_offset + (CTZLU(cur_word) / 2);
diff = new_missing_idx - last_missing_p1;
if (diff) {
dest = memcpya(dest, &(source[last_missing_p1 * elem_size]), diff * elem_size);
}
last_missing_p1 = new_missing_idx + 1;
cur_word &= cur_word - 1;
}
sample_idx_offset += BITCT2;
} while (loadbuf < loadbuf_end);
diff = unfiltered_sample_ct - last_missing_p1;
if (diff) {
memcpy(dest, &(source[last_missing_p1 * elem_size]), diff * elem_size);
}
}
uint32_t collapse_duplicate_ids(char* sorted_ids, uintptr_t id_ct, uintptr_t max_id_len, uint32_t* id_starts) {
// Collapses array of sorted IDs to remove duplicates, and writes
// pre-collapse positions to id_starts (so e.g. duplication count of any
// sample ID can be determined via subtraction) if it isn't nullptr.
// Returns id_ct of collapsed array.
uintptr_t read_idx;
uintptr_t write_idx;
if (!id_ct) {
return 0;
}
if (id_starts) {
id_starts[0] = 0;
for (read_idx = 1; read_idx < id_ct; read_idx++) {
if (!strcmp(&(sorted_ids[(read_idx - 1) * max_id_len]), &(sorted_ids[read_idx * max_id_len]))) {
break;
}
id_starts[read_idx] = read_idx;
}
write_idx = read_idx;
while (++read_idx < id_ct) {
if (strcmp(&(sorted_ids[(write_idx - 1) * max_id_len]), &(sorted_ids[read_idx * max_id_len]))) {
strcpy(&(sorted_ids[write_idx * max_id_len]), &(sorted_ids[read_idx * max_id_len]));
id_starts[write_idx++] = read_idx;
}
}
} else {
for (read_idx = 1; read_idx < id_ct; read_idx++) {
if (!strcmp(&(sorted_ids[(read_idx - 1) * max_id_len]), &(sorted_ids[read_idx * max_id_len]))) {
break;
}
}
write_idx = read_idx;
while (++read_idx < id_ct) {
if (strcmp(&(sorted_ids[(write_idx - 1) * max_id_len]), &(sorted_ids[read_idx * max_id_len]))) {
strcpy(&(sorted_ids[write_idx * max_id_len]), &(sorted_ids[read_idx * max_id_len]));
write_idx++;
}
}
}
return write_idx;
}
void range_list_init(Range_list* range_list_ptr) {
range_list_ptr->names = nullptr;
range_list_ptr->starts_range = nullptr;
range_list_ptr->name_ct = 0;
range_list_ptr->name_max_len = 0;
}
void free_range_list(Range_list* range_list_ptr) {
free_cond(range_list_ptr->names);
free_cond(range_list_ptr->starts_range);
}
// implementation used in PLINK 1.07 stats.cpp
// probably want to remove this function and use erf() calls in the future
double normdist(double zz) {
double sqrt2pi = 2.50662827463;
double t0;
double z1;
double p0;
t0 = 1 / (1 + 0.2316419 * fabs(zz));
z1 = exp(-0.5 * zz * zz) / sqrt2pi;
p0 = z1 * t0 * (0.31938153 + t0 * (-0.356563782 + t0 * (1.781477937 + t0 * (-1.821255978 + 1.330274429 * t0))));
return zz >= 0 ? 1 - p0 : p0;
}
double rand_normal(double* secondval_ptr) {
// N(0, 1)
double dxx = sqrt(-2 * log(rand_unif()));
double dyy = 2 * PI * rand_unif();
*secondval_ptr = dxx * cos(dyy);
return dxx * sin(dyy);
}
void init_sfmt64_from_sfmt32(sfmt_t* sfmt32, sfmt_t* sfmt64) {
// sfmt_genrand_uint64() is not supposed to be called after
// sfmt_genrand_uint32() is called on the same generator. To work around
// this, we initialize a new sfmt64 generator with this function when
// necessary, and stick to genrand_uint32() calls with the main generator.
uint32_t init_arr[4];
uint32_t uii;
for (uii = 0; uii < 4; uii++) {
init_arr[uii] = sfmt_genrand_uint32(sfmt32);
}
sfmt_init_by_array(sfmt64, init_arr, 4);
}
void generate_perm1_interleaved(uint32_t tot_ct, uint32_t set_ct, uintptr_t perm_idx, uintptr_t perm_ct, uintptr_t* perm_buf) {
uintptr_t tot_ctl = BITCT_TO_WORDCT(tot_ct);
uintptr_t tot_rem = tot_ct & (BITCT - 1);
uint32_t tot_quotient = (uint32_t)(0x100000000LLU / tot_ct);
uint32_t upper_bound = tot_ct * tot_quotient - 1;
uintptr_t uljj = perm_ct - perm_idx;
uint32_t totq_preshift;
uint64_t totq_magic;
uint32_t totq_postshift;
uint32_t totq_incr;
uintptr_t* pbptr;
uint32_t num_set;
uint32_t urand;
uintptr_t ulii;
// seeing as how we're gonna divide by the same number a billion times or so,
// it just might be worth optimizing that division...
magic_num(tot_quotient, &totq_magic, &totq_preshift, &totq_postshift, &totq_incr);
if (set_ct * 2 < tot_ct) {
for (ulii = 0; ulii < tot_ctl; ulii++) {
fill_ulong_zero(uljj, &(perm_buf[perm_idx + (ulii * perm_ct)]));
}
for (; perm_idx < perm_ct; perm_idx++) {
pbptr = &(perm_buf[perm_idx]);
for (num_set = 0; num_set < set_ct; num_set++) {
do {
do {
urand = sfmt_genrand_uint32(&g_sfmt);
} while (urand > upper_bound);
// this is identical to ulii = urand / tot_quotient
ulii = (totq_magic * ((urand >> totq_preshift) + totq_incr)) >> totq_postshift;
uljj = ulii / BITCT;
ulii &= (BITCT - 1);
} while ((pbptr[uljj * perm_ct] >> ulii) & 1);
pbptr[uljj * perm_ct] |= (ONELU << ulii);
}
}
} else {
for (ulii = 0; ulii < tot_ctl; ulii++) {
fill_ulong_one(uljj, &(perm_buf[perm_idx + (ulii * perm_ct)]));
}
// "set" has reversed meaning here
set_ct = tot_ct - set_ct;
for (; perm_idx < perm_ct; perm_idx++) {
pbptr = &(perm_buf[perm_idx]);
for (num_set = 0; num_set < set_ct; num_set++) {
do {
do {
urand = sfmt_genrand_uint32(&g_sfmt);
} while (urand > upper_bound);
ulii = (totq_magic * ((urand >> totq_preshift) + totq_incr)) >> totq_postshift;
uljj = ulii / BITCT;
ulii &= (BITCT - 1);
} while (!((pbptr[uljj * perm_ct] >> ulii) & 1));
pbptr[uljj * perm_ct] &= ~(ONELU << ulii);
}
}
if (tot_rem) {
uljj = (~ZEROLU) >> (BITCT - tot_rem);
pbptr = &(perm_buf[(tot_ctl - 1) * perm_ct + perm_idx]);
for (ulii = perm_idx; ulii < perm_ct; ulii++) {
*pbptr &= uljj;
pbptr++;
}
}
}
}
uint32_t cubic_real_roots(double coef_a, double coef_b, double coef_c, double* solutions) {
// Analytically finds all real roots of x^3 + ax^2 + bx + c, saving them in
// solutions[] (sorted from smallest to largest), and returning the count.
// Multiple roots are only returned/counted once.
// Additional research into numerical stability may be in order here.
double a2 = coef_a * coef_a;
double qq = (a2 - 3 * coef_b) * (1.0 / 9.0);
double rr = (2 * a2 * coef_a - 9 * coef_a * coef_b + 27 * coef_c) * (1.0 / 54.0);
double r2 = rr * rr;
double q3 = qq * qq * qq;
double adiv3 = coef_a * (1.0 / 3.0);
double sq;
double dxx;
if (r2 < q3) {
// three real roots
sq = sqrt(qq);
dxx = acos(rr / (qq * sq)) * (1.0 / 3.0);
sq *= -2;
solutions[0] = sq * cos(dxx) - adiv3;
solutions[1] = sq * cos(dxx + (2.0 * PI / 3.0)) - adiv3;
solutions[2] = sq * cos(dxx - (2.0 * PI / 3.0)) - adiv3;
// now sort and check for within-epsilon equality
if (solutions[0] > solutions[1]) {
dxx = solutions[0];
solutions[0] = solutions[1];
if (dxx > solutions[2]) {
solutions[1] = solutions[2];
solutions[2] = dxx;
} else {
solutions[1] = dxx;
}
if (solutions[0] > solutions[1]) {
dxx = solutions[0];
solutions[0] = solutions[1];
solutions[1] = dxx;
}
} else if (solutions[1] > solutions[2]) {
dxx = solutions[1];
solutions[1] = solutions[2];
solutions[2] = dxx;
}
if (solutions[1] - solutions[0] < EPSILON) {
solutions[1] = solutions[2];
return (solutions[1] - solutions[0] < EPSILON)? 1 : 2;
}
return (solutions[2] - solutions[1] < EPSILON)? 2 : 3;
}
dxx = -pow(fabs(rr) + sqrt(r2 - q3), 1.0 / 3.0);
if (dxx == 0.0) {
solutions[0] = -adiv3;
return 1;
}
if (rr < 0.0) {
dxx = -dxx;
}
sq = qq / dxx;
solutions[0] = dxx + sq - adiv3;
// use of regular epsilon here has actually burned us
if (fabs(dxx - sq) >= (EPSILON * 8)) {
return 1;
}
if (dxx >= 0.0) {
solutions[1] = solutions[0];
solutions[0] = -dxx - adiv3;
} else {
solutions[1] = -dxx - adiv3;
}
return 2;
}
void join_threads(pthread_t* threads, uint32_t ctp1) {
if (!(--ctp1)) {
return;
}
#ifdef _WIN32
WaitForMultipleObjects(ctp1, threads, 1, INFINITE);
for (uint32_t uii = 0; uii < ctp1; ++uii) {
CloseHandle(threads[uii]);
}
#else
for (uint32_t uii = 0; uii < ctp1; uii++) {
pthread_join(threads[uii], nullptr);
}
#endif
}
#ifdef _WIN32
int32_t spawn_threads(pthread_t* threads, unsigned (__stdcall *start_routine)(void*), uintptr_t ct)
#else
int32_t spawn_threads(pthread_t* threads, void* (*start_routine)(void*), uintptr_t ct)
#endif
{
uintptr_t ulii;
if (ct == 1) {
return 0;
}
for (ulii = 1; ulii < ct; ulii++) {
#ifdef _WIN32
threads[ulii - 1] = (HANDLE)_beginthreadex(nullptr, 4096, start_routine, (void*)ulii, 0, nullptr);
if (!threads[ulii - 1]) {
join_threads(threads, ulii);
return -1;
}
#else
if (pthread_create(&(threads[ulii - 1]), nullptr, start_routine, (void*)ulii)) {
join_threads(threads, ulii);
return -1;
}
#endif
}
return 0;
}
// Okay, it's time to bite the bullet and stop creating and destroying threads
// like crazy, at least in the small-block-size GRM calculation; Intel
// MKL-powered GCTA 1.24 blew away our code on the NIH 512-core test machine
// when the maximum number of threads was used. Mostly because threads were
// actually costing much more in creation/destruction time than they saved;
// much better wall-clock times would have resulted from manually setting
// --threads to a low number. That's not cool.
//
// New framework:
// * On all operating systems, g_is_last_thread_block indicates whether all
// threads should terminate upon completion of the current block. (Initially
// had this volatile, then realized that the presence of the sync-wait should
// be enough to force the global variable to be reread.)
// * On Linux and OS X, if we aren't dealing with the final block,
// spawn_threads2() also reinitializes g_thread_active_ct.
// * On Linux and OS X, spawn_threads2() checks if g_thread_mutex_initialized
// is set. If not, it, it is set, g_thread_sync_mutex,
// g_thread_cur_block_done_condvar and g_thread_start_next_condvar are
// initialized, then threads are launched.
// If it has, pthread_cond_broadcast() acts on g_thread_start_next_condvar.
// * On Windows, spawn_threads2() checks if g_thread_mutex_initialized is set.
// If it has not, it, along with g_thread_start_next_event[] and
// g_thread_cur_block_done_events[], are initialized, then the threads are
// launched. If it has, SetEvent() acts on g_thread_start_next_event[].
// (It used to act on only one event; then I realized that safely dealing
// with a manual-reset event could be a pain if the first thread finishes
// before the last one wakes up...)
// * Thread functions are expected to be of the form
// THREAD_RET_TYPE function_name(void* arg) {
// uintptr_t tidx = (uintptr_t)arg;
// ...
// while (1) {
// ... // process current block
// if ((!tidx) || g_is_last_thread_block) {
// THREAD_RETURN;
// }
// THREAD_BLOCK_FINISH(tidx);
// }
// }
// * On Linux and OS X, THREAD_BLOCK_FINISH() acquires a mutex, decrements
// g_thread_active_ct, calls pthread_cond_signal() on
// g_thread_cur_block_done_condvar iff g_thread_active_ct is now zero, then
// unconditionally calls pthread_cond_wait on g_thread_start_next_condvar and
// the mutex.
// * On Windows, THREAD_BLOCK_FINISH() calls SetEvent() on
// g_thread_cur_block_done_events[tidx - 1], then waits on
// g_thread_start_next_event[tidx - 1].
// * If the termination variable is set, join_threads2() waits for all threads
// to complete, then cleans up all multithreading objects. Otherwise, on
// Linux and OS X, it acquires the mutex and calls pthread_cond_wait() on
// g_thread_cur_block_done_condvar and the mutex; and on Windows, it calls
// WaitForMultipleObjects() on g_thread_cur_block_done_events[].
// WaitForMultipleObjects has a 64 object limit, and for now it doesn't seem
// too important to use a for loop to handle more objects?... well, we can
// add that if anyone wants it, but for now the Windows thread limit is 65
// (the main thread isn't part of the wait).
//
// This is only very slightly better than the original approach on my old
// MacBook Pro (since threading overhead was never high to begin with, there
// being only 2 cores...), but the impact should be more noticeable on heavily
// multicore machines.
//
// The next performance improvement to make is double-buffering; tricky to
// estimate how much (if any) "consumption" the main I/O thread should be
// doing, though, so it may want a job queue to go with it.
uintptr_t g_thread_spawn_ct;
uint32_t g_is_last_thread_block = 0;
#ifdef _WIN32
HANDLE g_thread_start_next_event[MAX_THREADS];
HANDLE g_thread_cur_block_done_events[MAX_THREADS];
#else
static pthread_mutex_t g_thread_sync_mutex;
static pthread_cond_t g_thread_cur_block_done_condvar;
static pthread_cond_t g_thread_start_next_condvar;
uint32_t g_thread_active_ct;
void THREAD_BLOCK_FINISH(uintptr_t tidx) {
uintptr_t initial_spawn_ct = g_thread_spawn_ct;
pthread_mutex_lock(&g_thread_sync_mutex);
if (!(--g_thread_active_ct)) {
pthread_cond_signal(&g_thread_cur_block_done_condvar);
}
while (g_thread_spawn_ct == initial_spawn_ct) {
// spurious wakeup guard
pthread_cond_wait(&g_thread_start_next_condvar, &g_thread_sync_mutex);
}
pthread_mutex_unlock(&g_thread_sync_mutex);
}
#endif
static uint32_t g_thread_mutex_initialized = 0;
void join_threads2(pthread_t* threads, uint32_t ctp1, uint32_t is_last_block) {
uint32_t uii;
if (!(--ctp1)) {
if (is_last_block) {
// allow another multithreaded function to be called later
g_thread_mutex_initialized = 0;
}
return;
}
#ifdef _WIN32
if (!is_last_block) {
WaitForMultipleObjects(ctp1, g_thread_cur_block_done_events, 1, INFINITE);
} else {
WaitForMultipleObjects(ctp1, threads, 1, INFINITE);
for (uii = 0; uii < ctp1; uii++) {
CloseHandle(threads[uii]);
CloseHandle(g_thread_start_next_event[uii]);
CloseHandle(g_thread_cur_block_done_events[uii]);
}
g_thread_mutex_initialized = 0;
}
#else
if (!is_last_block) {
pthread_mutex_lock(&g_thread_sync_mutex);
while (g_thread_active_ct) {
pthread_cond_wait(&g_thread_cur_block_done_condvar, &g_thread_sync_mutex);
}
// keep mutex until next block loaded
} else {
for (uii = 0; uii < ctp1; uii++) {
pthread_join(threads[uii], nullptr);
}
// slightly inefficient if there are multiple multithreaded commands being
// run, but if different commands require different numbers of threads,
// optimizing this sort of thing away could introduce bugs...
pthread_mutex_destroy(&g_thread_sync_mutex);
pthread_cond_destroy(&g_thread_cur_block_done_condvar);
pthread_cond_destroy(&g_thread_start_next_condvar);
g_thread_mutex_initialized = 0;
}
#endif
}
#ifdef _WIN32
int32_t spawn_threads2(pthread_t* threads, unsigned (__stdcall *start_routine)(void*), uintptr_t ct, uint32_t is_last_block)
#else
int32_t spawn_threads2(pthread_t* threads, void* (*start_routine)(void*), uintptr_t ct, uint32_t is_last_block)
#endif
{
uintptr_t ulii;
// this needs to go before the ct == 1 check since start_routine() might need
// it
if (g_is_last_thread_block != is_last_block) {
// might save us an unnecessary memory write that confuses the cache
// coherency logic?
g_is_last_thread_block = is_last_block;
}
#ifdef _WIN32
if (!g_thread_mutex_initialized) {
g_thread_spawn_ct = 0;
g_thread_mutex_initialized = 1;
if (ct == 1) {
return 0;
}
for (ulii = 1; ulii < ct; ulii++) {
g_thread_start_next_event[ulii - 1] = CreateEvent(nullptr, FALSE, FALSE, nullptr);
g_thread_cur_block_done_events[ulii - 1] = CreateEvent(nullptr, FALSE, FALSE, nullptr);
}
for (ulii = 1; ulii < ct; ulii++) {
threads[ulii - 1] = (HANDLE)_beginthreadex(nullptr, 4096, start_routine, (void*)ulii, 0, nullptr);
if (!threads[ulii - 1]) {
if (ulii > 1) {
join_threads2(threads, ulii, is_last_block);
if (!is_last_block) {
for (uintptr_t uljj = 0; uljj < ulii - 1; ++uljj) {
CloseHandle(threads[uljj]);
}
}
}
if ((!is_last_block) || (ulii == 1)) {
for (uint32_t uii = 0; uii < ct - 1; ++uii) {
CloseHandle(g_thread_start_next_event[uii]);
CloseHandle(g_thread_cur_block_done_events[uii]);
}
g_thread_mutex_initialized = 0;
}
return -1;
}
}
} else {
g_thread_spawn_ct++;
for (ulii = 1; ulii < ct; ulii++) {
SetEvent(g_thread_start_next_event[ulii - 1]);
}
}
#else
if (!is_last_block) {
g_thread_active_ct = ct - 1;
}
if (!g_thread_mutex_initialized) {
g_thread_spawn_ct = 0; // tidx 0 may need to know modulus
g_thread_mutex_initialized = 1;
if (ct == 1) {
return 0;
}
if (pthread_mutex_init(&g_thread_sync_mutex, nullptr) ||
pthread_cond_init(&g_thread_cur_block_done_condvar, nullptr) ||
pthread_cond_init(&g_thread_start_next_condvar, nullptr)) {
return -1;
}
for (ulii = 1; ulii < ct; ulii++) {
if (pthread_create(&(threads[ulii - 1]), nullptr, start_routine, (void*)ulii)) {
if (ulii > 1) {
join_threads2(threads, ulii, is_last_block);
if (!is_last_block) {
for (uintptr_t uljj = 0; uljj < ulii - 1; ++uljj) {
pthread_cancel(threads[uljj]);
}
}
}
if ((!is_last_block) || (ulii == 1)) {
pthread_mutex_destroy(&g_thread_sync_mutex);
pthread_cond_destroy(&g_thread_cur_block_done_condvar);
pthread_cond_destroy(&g_thread_start_next_condvar);
g_thread_mutex_initialized = 0;
}
return -1;
}
}
} else {
g_thread_spawn_ct++;
if (ct == 1) {
return 0;
}
// still holding mutex
pthread_mutex_unlock(&g_thread_sync_mutex);
pthread_cond_broadcast(&g_thread_start_next_condvar);
}
#endif
return 0;
}
sfmt_t** g_sfmtp_arr;
uint32_t bigstack_init_sfmtp(uint32_t thread_ct) {
uint32_t uibuf[4];
uint32_t tidx;
uint32_t uii;
g_sfmtp_arr = (sfmt_t**)bigstack_alloc(thread_ct * sizeof(intptr_t));
if (!g_sfmtp_arr) {
return 1;
}
g_sfmtp_arr[0] = &g_sfmt;
if (thread_ct > 1) {
for (tidx = 1; tidx < thread_ct; tidx++) {
g_sfmtp_arr[tidx] = (sfmt_t*)bigstack_alloc(sizeof(sfmt_t));
if (!g_sfmtp_arr[tidx]) {
return 1;
}
for (uii = 0; uii < 4; uii++) {
uibuf[uii] = sfmt_genrand_uint32(&g_sfmt);
}
sfmt_init_by_array(g_sfmtp_arr[tidx], uibuf, 4);
}
}
return 0;
}