// This file is part of PLINK 1.9, copyright (C) 2005-2024 Shaun Purcell, // Christopher Chang. // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with this program. If not, see . #include "plink_common.h" #include "plink_filter.h" #include "plink_stats.h" #include "pigz.h" void oblig_missing_init(Oblig_missing_info* om_ip) { om_ip->cluster_ct = 0; om_ip->entry_ct = 0; om_ip->entries = nullptr; om_ip->cluster_ref_cts = nullptr; om_ip->sample_lookup = nullptr; om_ip->marker_fname = nullptr; om_ip->sample_fname = nullptr; } void oblig_missing_cleanup(Oblig_missing_info* om_ip) { if (om_ip->marker_fname) { free_cond(om_ip->entries); free_cond(om_ip->cluster_ref_cts); free_cond(om_ip->sample_lookup); free_cond(om_ip->marker_fname); free_cond(om_ip->sample_fname); om_ip->marker_fname = nullptr; } } static const char keep_str[] = "keep"; static const char keep_fam_str[] = "keep-fam"; static const char remove_str[] = "remove"; static const char remove_fam_str[] = "remove-fam"; const char* keep_or_remove_flag_str(uint32_t flags) { switch (flags) { case 0: return keep_str; case 1: return remove_str; case 2: return keep_fam_str; case 3: return remove_fam_str; } return nullptr; } int32_t keep_or_remove(char* fname, char* sorted_ids, uintptr_t sorted_ids_ct, uintptr_t max_id_len, uint32_t* id_map, uintptr_t unfiltered_ct, uintptr_t* exclude_arr, uintptr_t* exclude_ct_ptr, uint32_t flags, uint32_t allow_no_samples) { FILE* infile = nullptr; unsigned char* bigstack_mark = g_bigstack_base; uintptr_t* exclude_arr_new = nullptr; uintptr_t unfiltered_ctl = BITCT_TO_WORDCT(unfiltered_ct); uintptr_t duplicate_ct = 0; uintptr_t line_idx = 0; uint32_t do_exclude = flags & 1; uint32_t families_only = flags & 2; int32_t retval = 0; char* id_buf; char* bufptr0; int32_t ii; uint32_t unfiltered_idx; uint32_t cur_idx; uint32_t last_idx; if (bigstack_alloc_ul(unfiltered_ctl, &exclude_arr_new)) { goto keep_or_remove_ret_NOMEM; } if (do_exclude) { // need this to avoid --keep/--remove spurious duplicate warnings (ID could // have been removed by an earlier step), though we could switch that to // the already_seen strategy memcpy(exclude_arr_new, exclude_arr, unfiltered_ctl * sizeof(intptr_t)); } else { fill_all_bits(unfiltered_ct, exclude_arr_new); } if (fopen_checked(fname, "r", &infile)) { goto keep_or_remove_ret_OPEN_FAIL; } g_textbuf[MAXLINELEN - 1] = ' '; if (bigstack_alloc_c(max_id_len, &id_buf)) { goto keep_or_remove_ret_NOMEM; } while (fgets(g_textbuf, MAXLINELEN, infile) != nullptr) { line_idx++; if (!g_textbuf[MAXLINELEN - 1]) { snprintf(g_logbuf, LOGBUFLEN, "Error: Line %" PRIuPTR " of --%s file is pathologically long.\n", line_idx, keep_or_remove_flag_str(flags)); goto keep_or_remove_ret_INVALID_FORMAT_2; } bufptr0 = skip_initial_spaces(g_textbuf); if (is_eoln_kns(*bufptr0)) { continue; } if (!families_only) { if (bsearch_read_fam_indiv(bufptr0, sorted_ids, max_id_len, sorted_ids_ct, nullptr, &ii, id_buf)) { snprintf(g_logbuf, LOGBUFLEN, "Error: Line %" PRIuPTR " of --%s file has fewer tokens than expected.\n", line_idx, keep_or_remove_flag_str(flags)); goto keep_or_remove_ret_INVALID_FORMAT_2; } if (ii != -1) { unfiltered_idx = id_map[(uint32_t)ii]; if (!IS_SET(exclude_arr, unfiltered_idx)) { if (do_exclude) { if (IS_SET(exclude_arr_new, unfiltered_idx)) { duplicate_ct++; } else { SET_BIT(unfiltered_idx, exclude_arr_new); } } else { if (!IS_SET(exclude_arr_new, unfiltered_idx)) { duplicate_ct++; } else { CLEAR_BIT(unfiltered_idx, exclude_arr_new); } } } } } else { bsearch_fam(bufptr0, sorted_ids, max_id_len, sorted_ids_ct, &cur_idx, &last_idx, id_buf); ii = 0; while (cur_idx < last_idx) { unfiltered_idx = id_map[cur_idx++]; if (!IS_SET(exclude_arr, unfiltered_idx)) { if (do_exclude) { if (IS_SET(exclude_arr_new, unfiltered_idx)) { ii = 1; } else { SET_BIT(unfiltered_idx, exclude_arr_new); } } else { if (!IS_SET(exclude_arr_new, unfiltered_idx)) { ii = 1; } else { CLEAR_BIT(unfiltered_idx, exclude_arr_new); } } } } // only add one to duplicate_ct, instead of one per family member duplicate_ct += (uint32_t)ii; } } if (!feof(infile)) { goto keep_or_remove_ret_READ_FAIL; } memcpy(exclude_arr, exclude_arr_new, unfiltered_ctl * sizeof(intptr_t)); *exclude_ct_ptr = popcount_longs(exclude_arr, unfiltered_ctl); if ((*exclude_ct_ptr == unfiltered_ct) && (!allow_no_samples)) { LOGERRPRINTF("Error: No %s remaining after --%s.\n", g_species_plural, keep_or_remove_flag_str(flags)); goto keep_or_remove_ret_ALL_SAMPLES_EXCLUDED; } unfiltered_ct -= *exclude_ct_ptr; // now filtered count LOGPRINTF("--%s: %" PRIuPTR " %s remaining.\n", keep_or_remove_flag_str(flags), unfiltered_ct, species_str(unfiltered_ct)); if (duplicate_ct) { // "At least" since this does not count duplicate IDs absent from the .bim. LOGERRPRINTF("Warning: At least %" PRIuPTR " duplicate ID%s in --%s file.\n", duplicate_ct, (duplicate_ct == 1)? "" : "s", keep_or_remove_flag_str(flags)); } while (0) { keep_or_remove_ret_NOMEM: retval = RET_NOMEM; break; keep_or_remove_ret_OPEN_FAIL: retval = RET_OPEN_FAIL; break; keep_or_remove_ret_READ_FAIL: retval = RET_READ_FAIL; break; keep_or_remove_ret_INVALID_FORMAT_2: logerrprintb(); retval = RET_INVALID_FORMAT; break; keep_or_remove_ret_ALL_SAMPLES_EXCLUDED: retval = RET_ALL_SAMPLES_EXCLUDED; break; } bigstack_reset(bigstack_mark); fclose_cond(infile); return retval; } // backported from plink 2.0 int32_t snps_flag(const char* variant_ids, const uint32_t* variant_id_htable, const Range_list* snps_range_list_ptr, uint32_t raw_variant_ct, uintptr_t max_variant_id_blen, uintptr_t variant_id_htable_size, uint32_t do_exclude, uintptr_t* variant_exclude, uintptr_t* exclude_ct_ptr) { unsigned char* bigstack_mark = g_bigstack_base; int32_t reterr = 0; { const char* varid_strbox = snps_range_list_ptr->names; const unsigned char* starts_range = snps_range_list_ptr->starts_range; const uint32_t varid_ct = snps_range_list_ptr->name_ct; const uintptr_t varid_max_blen = snps_range_list_ptr->name_max_len; const uint32_t raw_variant_ctl = BITCT_TO_WORDCT(raw_variant_ct); uintptr_t* seen_uidxs; if (bigstack_calloc_ul(raw_variant_ctl, &seen_uidxs)) { goto snps_flag_ret_NOMEM; } uint32_t range_start_vidx = 0xffffffffU; for (uint32_t varid_idx = 0; varid_idx < varid_ct; ++varid_idx) { const char* cur_varid = &(varid_strbox[varid_idx * varid_max_blen]); uint32_t variant_uidx = id_htable_find(cur_varid, strlen(cur_varid), variant_id_htable, variant_id_htable_size, variant_ids, max_variant_id_blen); if (variant_uidx == 0xffffffffU) { snprintf(g_logbuf, LOGBUFLEN, "Error: --%ssnps variant '%s' not found.\n", do_exclude? "exclude-" : "", cur_varid); goto snps_flag_ret_INVALID_FORMAT_WW; } if (starts_range[varid_idx]) { range_start_vidx = variant_uidx; } else { if (range_start_vidx != 0xffffffffU) { if (variant_uidx < range_start_vidx) { const uint32_t uii = variant_uidx; variant_uidx = range_start_vidx; range_start_vidx = uii; } fill_bits(range_start_vidx, variant_uidx + 1 - range_start_vidx, seen_uidxs); } else { set_bit(variant_uidx, seen_uidxs); } range_start_vidx = 0xffffffffU; } } if (do_exclude) { bitvec_or(seen_uidxs, raw_variant_ctl, variant_exclude); } else { bitvec_ornot(seen_uidxs, raw_variant_ctl, variant_exclude); zero_trailing_bits(raw_variant_ct, variant_exclude); } const uint32_t new_exclude_ct = popcount_longs(variant_exclude, raw_variant_ctl); const uint32_t new_variant_ct = raw_variant_ct - new_exclude_ct; LOGPRINTF("--%ssnps: %u variant%s remaining.\n", do_exclude? "exclude-" : "", new_variant_ct, (new_variant_ct == 1)? "" : "s"); *exclude_ct_ptr = new_exclude_ct; } while (0) { snps_flag_ret_NOMEM: reterr = RET_NOMEM; break; snps_flag_ret_INVALID_FORMAT_WW: wordwrapb(0); logerrprintb(); reterr = RET_INVALID_FORMAT; break; } bigstack_reset(bigstack_mark); return reterr; } void extract_exclude_process_token(const char* tok_start, const uint32_t* marker_id_htable, uint32_t marker_id_htable_size, const uint32_t* extra_alloc_base, const char* marker_ids, uintptr_t max_marker_id_len, uintptr_t* marker_exclude, uintptr_t* already_seen, uintptr_t* duplicate_ct_ptr, uint32_t do_exclude, uint32_t curtoklen) { if (curtoklen >= max_marker_id_len) { return; } uint32_t hashval = murmurhash3_32(tok_start, curtoklen) % marker_id_htable_size; uint32_t next_incr = 1; uint32_t cur_llidx = 0; const char* sptr; uintptr_t marker_uidx; uint32_t hash_result; uint32_t top_diff; uint32_t cur_dup; while (1) { hash_result = marker_id_htable[hashval]; cur_dup = hash_result >> 31; if (cur_dup) { if (hash_result == 0xffffffffU) { return; } cur_llidx = hash_result << 1; marker_uidx = extra_alloc_base[cur_llidx]; } else { // no need to check without --update-map, which requires uniqueness if (is_set(marker_exclude, hash_result)) { goto extract_exclude_process_token_next_hashval; } marker_uidx = hash_result; } sptr = &(marker_ids[marker_uidx * max_marker_id_len]); if ((!memcmp(tok_start, sptr, curtoklen)) && (!sptr[curtoklen])) { // when there are multiple variants with the same ID, only need to // check bit for first search result if (IS_SET(already_seen, marker_uidx)) { *duplicate_ct_ptr += 1; } else { SET_BIT(marker_uidx, already_seen); if (!cur_dup) { return; } while (1) { cur_llidx = extra_alloc_base[cur_llidx + 1]; if (cur_llidx == 0xffffffffU) { return; } SET_BIT(extra_alloc_base[cur_llidx], already_seen); } } } extract_exclude_process_token_next_hashval: top_diff = marker_id_htable_size - hashval; if (top_diff > next_incr) { hashval += next_incr; } else { hashval = next_incr - top_diff; } next_incr += 2; } } int32_t extract_exclude_flag_norange(char* fname, uint32_t* marker_id_htable, uint32_t marker_id_htable_size, uint32_t do_exclude, char* marker_ids, uintptr_t max_marker_id_len, uintptr_t unfiltered_marker_ct, uintptr_t* marker_exclude, uintptr_t* marker_exclude_ct_ptr, uint32_t allow_no_variants) { unsigned char* bigstack_mark = g_bigstack_base; FILE* infile = nullptr; uintptr_t unfiltered_marker_ctl = BITCT_TO_WORDCT(unfiltered_marker_ct); uintptr_t duplicate_ct = 0; // needs to be synced with populate_id_htable const uint32_t* extra_alloc_base = &(marker_id_htable[round_up_pow2_ui(marker_id_htable_size, CACHELINE_INT32)]); char* midbuf = &(g_textbuf[MAXLINELEN]); uint32_t curtoklen = 0; int32_t retval = 0; uintptr_t bufsize; uintptr_t* already_seen; char* bufptr0; char* bufptr; char* bufptr2; char* bufptr3; if (bigstack_calloc_ul(unfiltered_marker_ctl, &already_seen)) { goto extract_exclude_flag_norange_ret_NOMEM; } if (fopen_checked(fname, FOPEN_RB, &infile)) { goto extract_exclude_flag_norange_ret_OPEN_FAIL; } while (1) { if (fread_checked(midbuf, MAXLINELEN, infile, &bufsize)) { goto extract_exclude_flag_norange_ret_READ_FAIL; } if (!bufsize) { if (curtoklen) { g_textbuf[MAXLINELEN] = ' '; // bugfix for no-\n-on-last-line case extract_exclude_process_token(&(g_textbuf[MAXLINELEN - curtoklen]), marker_id_htable, marker_id_htable_size, extra_alloc_base, marker_ids, max_marker_id_len, marker_exclude, already_seen, &duplicate_ct, do_exclude, curtoklen); } break; } bufptr0 = &(midbuf[bufsize]); *bufptr0 = ' '; bufptr0[1] = '0'; bufptr = &(g_textbuf[MAXLINELEN - curtoklen]); bufptr2 = midbuf; if (curtoklen) { goto extract_exclude_flag_norange_tok_start; } while (1) { while (*bufptr <= ' ') { bufptr++; } if (bufptr >= bufptr0) { curtoklen = 0; break; } bufptr2 = &(bufptr[1]); extract_exclude_flag_norange_tok_start: while (*bufptr2 > ' ') { bufptr2++; } curtoklen = (uintptr_t)(bufptr2 - bufptr); if (bufptr2 == &(g_textbuf[MAXLINELEN * 2])) { if (curtoklen > MAXLINELEN) { snprintf(g_logbuf, LOGBUFLEN, "Error: Excessively long ID in --%s file.\n", do_exclude? "exclude" : "extract"); goto extract_exclude_flag_norange_ret_INVALID_FORMAT_2; } bufptr3 = &(g_textbuf[MAXLINELEN - curtoklen]); memcpy(bufptr3, bufptr, curtoklen); break; } extract_exclude_process_token(bufptr, marker_id_htable, marker_id_htable_size, extra_alloc_base, marker_ids, max_marker_id_len, marker_exclude, already_seen, &duplicate_ct, do_exclude, curtoklen); bufptr = &(bufptr2[1]); } } if (!feof(infile)) { goto extract_exclude_flag_norange_ret_READ_FAIL; } if (do_exclude) { bitvec_or(already_seen, unfiltered_marker_ctl, marker_exclude); } else { bitvec_ornot(already_seen, unfiltered_marker_ctl, marker_exclude); zero_trailing_bits(unfiltered_marker_ct, marker_exclude); } *marker_exclude_ct_ptr = popcount_longs(marker_exclude, unfiltered_marker_ctl); if ((*marker_exclude_ct_ptr == unfiltered_marker_ct) && (!allow_no_variants)) { LOGERRPRINTF("Error: No variants remaining after --%s.\n", do_exclude? "exclude" : "extract"); goto extract_exclude_flag_norange_ret_ALL_MARKERS_EXCLUDED; } unfiltered_marker_ct -= *marker_exclude_ct_ptr; // now filtered count LOGPRINTF("--%s: %" PRIuPTR " variant%s remaining.\n", do_exclude? "exclude" : "extract", unfiltered_marker_ct, (unfiltered_marker_ct == 1)? "" : "s"); if (duplicate_ct) { // "At least" since this does not count duplicate IDs absent from the .bim. LOGERRPRINTF("Warning: At least %" PRIuPTR " duplicate ID%s in --%s file.\n", duplicate_ct, (duplicate_ct == 1)? "" : "s", do_exclude? "exclude" : "extract"); } while (0) { extract_exclude_flag_norange_ret_NOMEM: retval = RET_NOMEM; break; extract_exclude_flag_norange_ret_OPEN_FAIL: retval = RET_OPEN_FAIL; break; extract_exclude_flag_norange_ret_READ_FAIL: retval = RET_READ_FAIL; break; extract_exclude_flag_norange_ret_INVALID_FORMAT_2: logerrprintb(); retval = RET_INVALID_FORMAT; break; extract_exclude_flag_norange_ret_ALL_MARKERS_EXCLUDED: retval = RET_ALL_MARKERS_EXCLUDED; break; } bigstack_reset(bigstack_mark); fclose_cond(infile); return retval; } int32_t filter_attrib(char* fname, char* condition_str, uint32_t* id_htable, uint32_t id_htable_size, uint32_t allow_no_variants, char* item_ids, uintptr_t max_id_len, uintptr_t unfiltered_ct, uintptr_t* exclude_arr, uintptr_t* exclude_ct_ptr) { gzFile gz_infile = nullptr; unsigned char* bigstack_mark = g_bigstack_base; uintptr_t include_ct = 0; uintptr_t unfiltered_ctl = BITCT_TO_WORDCT(unfiltered_ct); char* sorted_pos_match = nullptr; char* sorted_neg_match = nullptr; char* bufptr2 = nullptr; uint32_t pos_match_ct = 0; uint32_t neg_match_ct = 0; uintptr_t max_pos_match_len = 0; uintptr_t max_neg_match_len = 0; uintptr_t line_idx = 0; uint32_t is_neg = 0; int32_t retval = 0; uintptr_t* exclude_arr_new; uintptr_t* already_seen; char* loadbuf; char* cond_ptr; char* bufptr; uintptr_t loadbuf_size; uintptr_t pos_match_idx; uintptr_t neg_match_idx; uintptr_t ulii; uint32_t item_uidx; uint32_t pos_match_needed; int32_t sorted_idx; if (bigstack_alloc_ul(unfiltered_ctl, &exclude_arr_new) || bigstack_calloc_ul(unfiltered_ctl, &already_seen)) { goto filter_attrib_ret_NOMEM; } fill_all_bits(unfiltered_ct, exclude_arr_new); if (condition_str) { // allow nullptr condition_str; this means all samples/variants named in // the file are included cond_ptr = condition_str; while (1) { while (*cond_ptr == ',') { cond_ptr++; } if (*cond_ptr == '-') { cond_ptr++; if (*cond_ptr == ',') { continue; } else if (*cond_ptr == '-') { logerrprint("Error: --attrib condition cannot contain consecutive dashes.\n"); goto filter_attrib_ret_INVALID_CMDLINE; } is_neg = 1; } else if (!(*cond_ptr)) { // command-line parser actually comma-terminates the string for us, so // no need to check for null elsewhere break; } bufptr = strchr(cond_ptr, ','); ulii = (uintptr_t)(bufptr - cond_ptr); if (is_neg) { neg_match_ct++; if (ulii >= max_neg_match_len) { max_neg_match_len = ulii + 1; } } else { pos_match_ct++; if (ulii >= max_pos_match_len) { max_pos_match_len = ulii + 1; } } cond_ptr = bufptr; is_neg = 0; } if (pos_match_ct) { if (bigstack_alloc_c(max_pos_match_len * pos_match_ct, &sorted_pos_match)) { goto filter_attrib_ret_NOMEM; } } if (neg_match_ct) { if (bigstack_alloc_c(max_neg_match_len * neg_match_ct, &sorted_neg_match)) { goto filter_attrib_ret_NOMEM; } } pos_match_idx = 0; neg_match_idx = 0; cond_ptr = condition_str; is_neg = 0; while (1) { while (*cond_ptr == ',') { cond_ptr++; } if (*cond_ptr == '-') { cond_ptr++; if (*cond_ptr == ',') { continue; } is_neg = 1; } else if (!(*cond_ptr)) { break; } bufptr = strchr(cond_ptr, ','); ulii = (uintptr_t)(bufptr - cond_ptr); if (is_neg) { memcpyx(&(sorted_neg_match[neg_match_idx * max_neg_match_len]), cond_ptr, ulii, '\0'); neg_match_idx++; } else { memcpyx(&(sorted_pos_match[pos_match_idx * max_pos_match_len]), cond_ptr, ulii, '\0'); pos_match_idx++; } cond_ptr = bufptr; is_neg = 0; } if (pos_match_ct) { qsort(sorted_pos_match, pos_match_ct, max_pos_match_len, strcmp_casted); bufptr = scan_for_duplicate_ids(sorted_pos_match, pos_match_ct, max_pos_match_len); if (bufptr) { LOGPREPRINTFWW("Error: Duplicate attribute '%s' in --attrib argument.\n", bufptr); goto filter_attrib_ret_INVALID_CMDLINE_2; } } if (neg_match_ct) { qsort(sorted_neg_match, neg_match_ct, max_neg_match_len, strcmp_casted); // bugfix: when multiple negative match conditions are present, presence // of ANY of them is enough to disqualify the variant. So it's // appropriate to ensure no duplication between positive and negative // match conditions after all. bufptr = scan_for_duplicate_or_overlap_ids(sorted_neg_match, neg_match_ct, max_neg_match_len, sorted_pos_match, pos_match_ct, max_pos_match_len); if (bufptr) { LOGPREPRINTFWW("Error: Duplicate attribute '%s' in --attrib argument.\n", bufptr); goto filter_attrib_ret_INVALID_CMDLINE_2; } } } loadbuf_size = bigstack_left(); if (loadbuf_size > MAXLINEBUFLEN) { loadbuf_size = MAXLINEBUFLEN; } else if (loadbuf_size <= MAXLINELEN) { goto filter_attrib_ret_NOMEM; } retval = gzopen_read_checked(fname, &gz_infile); if (retval) { goto filter_attrib_ret_1; } loadbuf = (char*)g_bigstack_base; loadbuf[loadbuf_size - 1] = ' '; while (1) { line_idx++; if (!gzgets(gz_infile, loadbuf, loadbuf_size)) { if (!gzeof(gz_infile)) { goto filter_attrib_ret_READ_FAIL; } break; } if (!loadbuf[loadbuf_size - 1]) { if (loadbuf_size == MAXLINEBUFLEN) { snprintf(g_logbuf, LOGBUFLEN, "Error: Line %" PRIuPTR" of --attrib file is pathologically long.\n", line_idx); goto filter_attrib_ret_INVALID_FORMAT_2; } goto filter_attrib_ret_NOMEM; } bufptr = skip_initial_spaces(loadbuf); if (is_eoln_kns(*bufptr)) { continue; } bufptr2 = token_endnn(bufptr); cond_ptr = skip_initial_spaces(bufptr2); item_uidx = id_htable_find(bufptr, (uintptr_t)(bufptr2 - bufptr), id_htable, id_htable_size, item_ids, max_id_len); if ((item_uidx == 0xffffffffU) || is_set(exclude_arr, item_uidx)) { // miss_ct++; continue; } if (is_set(already_seen, item_uidx)) { *bufptr2 = '\0'; LOGPREPRINTFWW("Error: Duplicate variant ID '%s' in --attrib file.\n", bufptr); goto filter_attrib_ret_INVALID_FORMAT_2; } set_bit(item_uidx, already_seen); pos_match_needed = pos_match_ct; while (!is_eoln_kns(*cond_ptr)) { bufptr2 = cond_ptr; bufptr = token_endnn(cond_ptr); ulii = (uintptr_t)(bufptr - bufptr2); cond_ptr = skip_initial_spaces(bufptr); if (pos_match_needed && (bsearch_str(bufptr2, ulii, sorted_pos_match, max_pos_match_len, pos_match_ct) != -1)) { pos_match_needed = 0; } else if (neg_match_ct) { sorted_idx = bsearch_str(bufptr2, ulii, sorted_neg_match, max_neg_match_len, neg_match_ct); if (sorted_idx != -1) { // fail pos_match_needed = 1; break; } } } if (pos_match_needed) { continue; } clear_bit(item_uidx, exclude_arr_new); include_ct++; } if ((!include_ct) && (!allow_no_variants)) { logerrprint("Error: No variants remaining after --attrib.\n"); retval = RET_ALL_MARKERS_EXCLUDED; goto filter_attrib_ret_1; } LOGPRINTF("--attrib: %" PRIuPTR " variant%s remaining.\n", include_ct, (include_ct == 1)? "" : "s"); memcpy(exclude_arr, exclude_arr_new, unfiltered_ctl * sizeof(intptr_t)); *exclude_ct_ptr = unfiltered_ct - include_ct; while (0) { filter_attrib_ret_NOMEM: retval = RET_NOMEM; break; filter_attrib_ret_READ_FAIL: retval = RET_READ_FAIL; break; filter_attrib_ret_INVALID_CMDLINE_2: logerrprintb(); filter_attrib_ret_INVALID_CMDLINE: retval = RET_INVALID_CMDLINE; break; filter_attrib_ret_INVALID_FORMAT_2: logerrprintb(); retval = RET_INVALID_FORMAT; break; } filter_attrib_ret_1: bigstack_reset(bigstack_mark); gzclose_cond(gz_infile); return retval; } int32_t filter_attrib_sample(char* fname, char* condition_str, char* sorted_ids, uintptr_t sorted_ids_ct, uintptr_t max_id_len, uint32_t* id_map, uintptr_t unfiltered_ct, uint32_t allow_no_samples, uintptr_t* exclude_arr, uintptr_t* exclude_ct_ptr) { // re-merge this with filter_attrib() after making sample ID lookup // hash-based gzFile gz_infile = nullptr; unsigned char* bigstack_mark = g_bigstack_base; uintptr_t include_ct = 0; uintptr_t unfiltered_ctl = BITCT_TO_WORDCT(unfiltered_ct); char* sorted_pos_match = nullptr; char* sorted_neg_match = nullptr; char* id_buf = nullptr; char* bufptr2 = nullptr; uint32_t pos_match_ct = 0; uint32_t neg_match_ct = 0; uintptr_t max_pos_match_len = 0; uintptr_t max_neg_match_len = 0; uintptr_t line_idx = 0; uint32_t is_neg = 0; int32_t retval = 0; uintptr_t* exclude_arr_new; uintptr_t* already_seen; char* loadbuf; char* cond_ptr; char* bufptr; uintptr_t loadbuf_size; uintptr_t pos_match_idx; uintptr_t neg_match_idx; uintptr_t ulii; uint32_t unfiltered_idx; uint32_t pos_match_needed; int32_t sorted_idx; if (bigstack_alloc_ul(unfiltered_ctl, &exclude_arr_new) || bigstack_calloc_ul(unfiltered_ctl, &already_seen) || bigstack_alloc_c(max_id_len, &id_buf)) { goto filter_attrib_sample_ret_NOMEM; } fill_all_bits(unfiltered_ct, exclude_arr_new); if (condition_str) { // allow nullptr condition_str; this means all samples/variants named in // the file are included cond_ptr = condition_str; while (1) { while (*cond_ptr == ',') { cond_ptr++; } if (*cond_ptr == '-') { cond_ptr++; if (*cond_ptr == ',') { continue; } else if (*cond_ptr == '-') { logerrprint("Error: --attrib-indiv condition cannot contain consecutive dashes.\n"); goto filter_attrib_sample_ret_INVALID_CMDLINE; } is_neg = 1; } else if (!(*cond_ptr)) { // command-line parser actually comma-terminates the string for us, so // no need to check for null elsewhere break; } bufptr = strchr(cond_ptr, ','); ulii = (uintptr_t)(bufptr - cond_ptr); if (is_neg) { neg_match_ct++; if (ulii >= max_neg_match_len) { max_neg_match_len = ulii + 1; } } else { pos_match_ct++; if (ulii >= max_pos_match_len) { max_pos_match_len = ulii + 1; } } cond_ptr = bufptr; is_neg = 0; } if (pos_match_ct) { if (bigstack_alloc_c(max_pos_match_len * pos_match_ct, &sorted_pos_match)) { goto filter_attrib_sample_ret_NOMEM; } } if (neg_match_ct) { if (bigstack_alloc_c(max_neg_match_len * neg_match_ct, &sorted_neg_match)) { goto filter_attrib_sample_ret_NOMEM; } } pos_match_idx = 0; neg_match_idx = 0; cond_ptr = condition_str; is_neg = 0; while (1) { while (*cond_ptr == ',') { cond_ptr++; } if (*cond_ptr == '-') { cond_ptr++; if (*cond_ptr == ',') { continue; } is_neg = 1; } else if (!(*cond_ptr)) { break; } bufptr = strchr(cond_ptr, ','); ulii = (uintptr_t)(bufptr - cond_ptr); if (is_neg) { memcpyx(&(sorted_neg_match[neg_match_idx * max_neg_match_len]), cond_ptr, ulii, '\0'); neg_match_idx++; } else { memcpyx(&(sorted_pos_match[pos_match_idx * max_pos_match_len]), cond_ptr, ulii, '\0'); pos_match_idx++; } cond_ptr = bufptr; is_neg = 0; } if (pos_match_ct) { qsort(sorted_pos_match, pos_match_ct, max_pos_match_len, strcmp_casted); bufptr = scan_for_duplicate_ids(sorted_pos_match, pos_match_ct, max_pos_match_len); if (bufptr) { LOGPREPRINTFWW("Error: Duplicate attribute '%s' in --attrib-indiv argument.\n", bufptr); goto filter_attrib_sample_ret_INVALID_CMDLINE_2; } } if (neg_match_ct) { qsort(sorted_neg_match, neg_match_ct, max_neg_match_len, strcmp_casted); bufptr = scan_for_duplicate_or_overlap_ids(sorted_neg_match, neg_match_ct, max_neg_match_len, sorted_pos_match, pos_match_ct, max_pos_match_len); if (bufptr) { LOGPREPRINTFWW("Error: Duplicate attribute '%s' in --attrib-indiv argument.\n", bufptr); goto filter_attrib_sample_ret_INVALID_CMDLINE_2; } } } loadbuf_size = bigstack_left(); if (loadbuf_size > MAXLINEBUFLEN) { loadbuf_size = MAXLINEBUFLEN; } else if (loadbuf_size <= MAXLINELEN) { goto filter_attrib_sample_ret_NOMEM; } retval = gzopen_read_checked(fname, &gz_infile); if (retval) { goto filter_attrib_sample_ret_1; } loadbuf = (char*)g_bigstack_base; loadbuf[loadbuf_size - 1] = ' '; while (1) { line_idx++; if (!gzgets(gz_infile, loadbuf, loadbuf_size)) { if (!gzeof(gz_infile)) { goto filter_attrib_sample_ret_READ_FAIL; } break; } if (!loadbuf[loadbuf_size - 1]) { if (loadbuf_size == MAXLINEBUFLEN) { snprintf(g_logbuf, LOGBUFLEN, "Error: Line %" PRIuPTR" of --attrib-indiv file is pathologically long.\n", line_idx); goto filter_attrib_sample_ret_INVALID_FORMAT_2; } goto filter_attrib_sample_ret_NOMEM; } bufptr = skip_initial_spaces(loadbuf); if (is_eoln_kns(*bufptr)) { continue; } if (bsearch_read_fam_indiv(bufptr, sorted_ids, max_id_len, sorted_ids_ct, &cond_ptr, &sorted_idx, id_buf)) { snprintf(g_logbuf, LOGBUFLEN, "Error: Line %" PRIuPTR " of --attrib-indiv file has fewer tokens than\nexpected.\n", line_idx); goto filter_attrib_sample_ret_INVALID_FORMAT_2; } if (sorted_idx == -1) { continue; } if (is_set(already_seen, sorted_idx)) { *strchr(id_buf, '\t') = ' '; LOGPREPRINTFWW("Error: Duplicate sample ID '%s' in --attrib-indiv file.\n", id_buf); goto filter_attrib_sample_ret_INVALID_FORMAT_2; } set_bit(sorted_idx, already_seen); unfiltered_idx = id_map[(uint32_t)sorted_idx]; if (is_set(exclude_arr, unfiltered_idx)) { // bugfix: don't proceed here continue; } pos_match_needed = pos_match_ct; while (!is_eoln_kns(*cond_ptr)) { bufptr2 = cond_ptr; bufptr = token_endnn(cond_ptr); ulii = (uintptr_t)(bufptr - bufptr2); cond_ptr = skip_initial_spaces(bufptr); if (pos_match_needed && (bsearch_str(bufptr2, ulii, sorted_pos_match, max_pos_match_len, pos_match_ct) != -1)) { pos_match_needed = 0; } else if (neg_match_ct) { sorted_idx = bsearch_str(bufptr2, ulii, sorted_neg_match, max_neg_match_len, neg_match_ct); if (sorted_idx != -1) { // fail pos_match_needed = 1; break; } } } if (pos_match_needed) { continue; } clear_bit(unfiltered_idx, exclude_arr_new); include_ct++; } if ((!include_ct) && (!allow_no_samples)) { LOGERRPRINTF("Error: No %s remaining after --attrib-indiv.\n", g_species_plural); retval = RET_ALL_SAMPLES_EXCLUDED; goto filter_attrib_sample_ret_1; } LOGPRINTF("--attrib-indiv: %" PRIuPTR " %s remaining.\n", include_ct, species_str(include_ct)); memcpy(exclude_arr, exclude_arr_new, unfiltered_ctl * sizeof(intptr_t)); *exclude_ct_ptr = unfiltered_ct - include_ct; while (0) { filter_attrib_sample_ret_NOMEM: retval = RET_NOMEM; break; filter_attrib_sample_ret_READ_FAIL: retval = RET_READ_FAIL; break; filter_attrib_sample_ret_INVALID_CMDLINE_2: logerrprintb(); filter_attrib_sample_ret_INVALID_CMDLINE: retval = RET_INVALID_CMDLINE; break; filter_attrib_sample_ret_INVALID_FORMAT_2: logerrprintb(); retval = RET_INVALID_FORMAT; break; } filter_attrib_sample_ret_1: bigstack_reset(bigstack_mark); gzclose_cond(gz_infile); return retval; } int32_t filter_qual_scores(Two_col_params* qual_filter, double qual_min_thresh, double qual_max_thresh, uint32_t* marker_id_htable, uint32_t marker_id_htable_size, uint32_t allow_no_variants, char* marker_ids, uintptr_t max_marker_id_len, uintptr_t unfiltered_marker_ct, uintptr_t* marker_exclude, uintptr_t* marker_exclude_ct_ptr) { unsigned char* bigstack_mark = g_bigstack_base; FILE* infile = nullptr; uintptr_t unfiltered_marker_ctl = BITCT_TO_WORDCT(unfiltered_marker_ct); uintptr_t miss_ct = 0; uint32_t varid_first = (qual_filter->colid < qual_filter->colx); char skipchar = qual_filter->skipchar; uintptr_t* already_seen; uintptr_t* marker_exclude_orig; char* loadbuf; uintptr_t loadbuf_size; uintptr_t slen; uintptr_t line_idx; uintptr_t marker_ct; char* colid_ptr; // variant ID char* colx_ptr; // quality score double dxx; uint32_t colmin; uint32_t coldiff; uint32_t marker_uidx; int32_t retval; char cc; if (bigstack_calloc_ul(unfiltered_marker_ctl, &already_seen) || bigstack_alloc_ul(unfiltered_marker_ctl, &marker_exclude_orig)) { goto filter_qual_scores_ret_NOMEM; } memcpy(marker_exclude_orig, marker_exclude, unfiltered_marker_ctl * sizeof(intptr_t)); loadbuf = (char*)g_bigstack_base; loadbuf_size = bigstack_left(); if (loadbuf_size > MAXLINEBUFLEN) { loadbuf_size = MAXLINEBUFLEN; } if (loadbuf_size <= MAXLINELEN) { goto filter_qual_scores_ret_NOMEM; } retval = open_and_skip_first_lines(&infile, qual_filter->fname, loadbuf, loadbuf_size, qual_filter->skip); if (retval) { goto filter_qual_scores_ret_1; } if (varid_first) { colmin = qual_filter->colid - 1; coldiff = qual_filter->colx - qual_filter->colid; } else { colmin = qual_filter->colx - 1; coldiff = qual_filter->colid - qual_filter->colx; } line_idx = qual_filter->skip; while (fgets(loadbuf, loadbuf_size, infile)) { line_idx++; if (!loadbuf[loadbuf_size - 1]) { if (loadbuf_size == MAXLINEBUFLEN) { snprintf(g_logbuf, LOGBUFLEN, "Error: Line %" PRIuPTR " of --qual-scores file is pathologically long.\n", line_idx); goto filter_qual_scores_ret_INVALID_FORMAT_2; } else { goto filter_qual_scores_ret_NOMEM; } } colid_ptr = skip_initial_spaces(loadbuf); cc = *colid_ptr; if (is_eoln_kns(cc) || (cc == skipchar)) { continue; } if (varid_first) { colid_ptr = next_token_multz(colid_ptr, colmin); colx_ptr = next_token_mult(colid_ptr, coldiff); if (no_more_tokens_kns(colx_ptr)) { goto filter_qual_scores_ret_MISSING_TOKENS; } } else { colx_ptr = next_token_multz(colid_ptr, colmin); colid_ptr = next_token_mult(colx_ptr, coldiff); if (no_more_tokens_kns(colid_ptr)) { goto filter_qual_scores_ret_MISSING_TOKENS; } } slen = strlen_se(colid_ptr); marker_uidx = id_htable_find(colid_ptr, slen, marker_id_htable, marker_id_htable_size, marker_ids, max_marker_id_len); if ((marker_uidx == 0xffffffffU) || is_set(marker_exclude_orig, marker_uidx)) { miss_ct++; continue; } if (is_set(already_seen, marker_uidx)) { colid_ptr[slen] = '\0'; LOGPREPRINTFWW("Error: Duplicate variant '%s' in --qual-scores file.\n", colid_ptr); goto filter_qual_scores_ret_INVALID_FORMAT_2; } set_bit(marker_uidx, already_seen); if (scan_double(colx_ptr, &dxx) || (dxx < qual_min_thresh) || (dxx > qual_max_thresh)) { set_bit(marker_uidx, marker_exclude); } } if (!feof(infile)) { goto filter_qual_scores_ret_READ_FAIL; } *marker_exclude_ct_ptr = popcount_longs(marker_exclude, unfiltered_marker_ctl); marker_ct = unfiltered_marker_ct - *marker_exclude_ct_ptr; if ((!marker_ct) && (!allow_no_variants)) { logerrprint("Error: No variants remaining after --qual-scores.\n"); retval = RET_ALL_MARKERS_EXCLUDED; goto filter_qual_scores_ret_1; } if (miss_ct) { snprintf(g_logbuf, LOGBUFLEN, "--qual-scores: %" PRIuPTR " variant%s remaining, %" PRIuPTR " ID%s missing.\n", marker_ct, (marker_ct == 1)? "" : "s", miss_ct, (miss_ct == 1)? "" : "s"); } else { snprintf(g_logbuf, LOGBUFLEN, "--qual-scores: %" PRIuPTR " variant%s remaining.\n", marker_ct, (marker_ct == 1)? "" : "s"); } logprintb(); while (0) { filter_qual_scores_ret_NOMEM: retval = RET_NOMEM; break; filter_qual_scores_ret_READ_FAIL: retval = RET_READ_FAIL; break; filter_qual_scores_ret_MISSING_TOKENS: snprintf(g_logbuf, LOGBUFLEN, "Error: Line %" PRIuPTR " of --qual-scores file has fewer tokens than expected.\n", line_idx); filter_qual_scores_ret_INVALID_FORMAT_2: logerrprintb(); retval = RET_INVALID_FORMAT; break; } filter_qual_scores_ret_1: bigstack_reset(bigstack_mark); fclose_cond(infile); return retval; } uint32_t random_thin_markers(double thin_keep_prob, uintptr_t unfiltered_marker_ct, uintptr_t* marker_exclude, uintptr_t* marker_exclude_ct_ptr, uint32_t allow_no_variants) { uint32_t marker_ct = unfiltered_marker_ct - *marker_exclude_ct_ptr; uint32_t marker_uidx = 0; uint32_t markers_done = 0; uint32_t removed_ct = 0; uint32_t uint32_thresh = (uint32_t)(thin_keep_prob * 4294967296.0 + 0.5); uint32_t marker_uidx_stop; while (markers_done < marker_ct) { marker_uidx = next_unset_unsafe(marker_exclude, marker_uidx); marker_uidx_stop = next_set(marker_exclude, marker_uidx, unfiltered_marker_ct); markers_done += marker_uidx_stop - marker_uidx; do { if (sfmt_genrand_uint32(&g_sfmt) >= uint32_thresh) { SET_BIT(marker_uidx, marker_exclude); removed_ct++; } } while (++marker_uidx < marker_uidx_stop); } if ((marker_ct == removed_ct) && (!allow_no_variants)) { logerrprint("Error: All variants removed by --thin. Try a higher probability.\n"); return 1; } LOGPRINTF("--thin: %u variant%s removed (%u remaining).\n", removed_ct, (removed_ct == 1)? "" : "s", marker_ct - removed_ct); *marker_exclude_ct_ptr += removed_ct; return 0; } int32_t random_thin_markers_ct(uint32_t thin_keep_ct, uintptr_t unfiltered_marker_ct, uintptr_t* marker_exclude, uintptr_t* marker_exclude_ct_ptr) { unsigned char* bigstack_mark = g_bigstack_base; uint32_t marker_ct = unfiltered_marker_ct - *marker_exclude_ct_ptr; uint32_t marker_uidx = 0; uintptr_t marker_ctl = BITCT_TO_WORDCT(marker_ct); int32_t retval = 0; uintptr_t* perm_buf; uint32_t marker_idx; if (thin_keep_ct > marker_ct) { LOGERRPRINTF("Error: --thin-count parameter exceeds number of remaining variants.\n"); goto random_thin_markers_ct_ret_INVALID_CMDLINE; } if (marker_ct > 1) { if (bigstack_alloc_ul(marker_ctl, &perm_buf)) { goto random_thin_markers_ct_ret_NOMEM; } // no actual interleaving here, but may as well use this function // note that this requires marker_ct >= 2 generate_perm1_interleaved(marker_ct, marker_ct - thin_keep_ct, 0, 1, perm_buf); marker_uidx = 0; for (marker_idx = 0; marker_idx < marker_ct; marker_uidx++, marker_idx++) { next_unset_unsafe_ck(marker_exclude, &marker_uidx); if (is_set(perm_buf, marker_idx)) { set_bit(marker_uidx, marker_exclude); } } } else if ((!thin_keep_ct) && marker_ct) { marker_uidx = next_unset_unsafe(marker_exclude, 0); set_bit(marker_uidx, marker_exclude); } LOGPRINTF("--thin-count: %u variant%s removed (%u remaining).\n", marker_ct - thin_keep_ct, (marker_ct - thin_keep_ct == 1)? "" : "s", thin_keep_ct); *marker_exclude_ct_ptr = unfiltered_marker_ct - thin_keep_ct; while (0) { random_thin_markers_ct_ret_NOMEM: retval = RET_NOMEM; break; random_thin_markers_ct_ret_INVALID_CMDLINE: retval = RET_INVALID_CMDLINE; break; } bigstack_reset(bigstack_mark); return retval; } uint32_t random_thin_samples(double thin_keep_prob, uintptr_t unfiltered_sample_ct, uint32_t allow_no_samples, uintptr_t* sample_exclude, uintptr_t* sample_exclude_ct_ptr) { uint32_t sample_ct = unfiltered_sample_ct - *sample_exclude_ct_ptr; uint32_t sample_uidx = 0; uint32_t samples_done = 0; uint32_t removed_ct = 0; uint32_t uint32_thresh = (uint32_t)(thin_keep_prob * 4294967296.0 + 0.5); uint32_t sample_uidx_stop; while (samples_done < sample_ct) { sample_uidx = next_unset_unsafe(sample_exclude, sample_uidx); sample_uidx_stop = next_set(sample_exclude, sample_uidx, unfiltered_sample_ct); samples_done += sample_uidx_stop - sample_uidx; do { if(sfmt_genrand_uint32(&g_sfmt) >= uint32_thresh) { SET_BIT(sample_uidx, sample_exclude); removed_ct++; } } while (++sample_uidx < sample_uidx_stop); } if ((sample_ct == removed_ct) && (!allow_no_samples)) { LOGERRPRINTF("Error: All %s removed by --thin-indiv. Try a higher probability.\n", g_species_plural); return 1; } LOGPRINTF("--thin-indiv: %u %s removed (%u remaining).\n", removed_ct, (removed_ct==1)? g_species_singular : g_species_plural, sample_ct - removed_ct); *sample_exclude_ct_ptr += removed_ct; return 0; } int32_t random_thin_samples_ct(uint32_t thin_keep_ct, uintptr_t unfiltered_sample_ct, uintptr_t* sample_exclude, uintptr_t* sample_exclude_ct_ptr) { unsigned char* bigstack_mark = g_bigstack_base; uint32_t sample_ct = unfiltered_sample_ct - *sample_exclude_ct_ptr; uint32_t sample_uidx = 0; uintptr_t sample_ctl = BITCT_TO_WORDCT(sample_ct); int32_t retval = 0; uintptr_t* perm_buf; uint32_t sample_idx; if (thin_keep_ct > sample_ct) { LOGERRPRINTF("Error: --thin-indiv-count parameter exceeds number of remaining %s.\n", g_species_plural); goto random_thin_samples_ct_ret_INVALID_CMDLINE; } if (bigstack_alloc_ul(sample_ctl, &perm_buf)) { goto random_thin_samples_ct_ret_NOMEM; } generate_perm1_interleaved(sample_ct, sample_ct - thin_keep_ct, 0, 1, perm_buf); sample_uidx = 0; for (sample_idx = 0; sample_idx < sample_ct; sample_uidx++, sample_idx++) { next_unset_unsafe_ck(sample_exclude, &sample_uidx); if (is_set(perm_buf, sample_idx)) { set_bit(sample_uidx, sample_exclude); } } LOGPRINTF("--thin-indiv-count: %u %s removed (%u remaining).\n", sample_ct - thin_keep_ct, (sample_ct - thin_keep_ct == 1)? g_species_singular : g_species_plural, thin_keep_ct); *sample_exclude_ct_ptr = unfiltered_sample_ct - thin_keep_ct; while(0) { random_thin_samples_ct_ret_NOMEM: retval = RET_NOMEM; break; random_thin_samples_ct_ret_INVALID_CMDLINE: retval = RET_INVALID_CMDLINE; break; } bigstack_reset(bigstack_mark); return retval; } int32_t load_oblig_missing(FILE* bedfile, uintptr_t bed_offset, uintptr_t unfiltered_marker_ct, uintptr_t* marker_exclude, uintptr_t marker_exclude_ct, char* marker_ids, uintptr_t max_marker_id_len, char* sorted_sample_ids, uintptr_t sorted_sample_ct, uintptr_t max_sample_id_len, uint32_t* sample_id_map, uintptr_t unfiltered_sample_ct, uintptr_t* sex_male, Chrom_info* chrom_info_ptr, Oblig_missing_info* om_ip) { // 1. load and validate cluster file // 2. load marker file, sort by uidx // 3. check for early exit (no clusters and/or no .zero entries) // 4. scan through .bed sequentially, update oblig_missing_..._cts unsigned char* bigstack_mark = g_bigstack_base; unsigned char* bigstack_end_mark = g_bigstack_end; FILE* infile = nullptr; char* idbuf = &(g_textbuf[MAXLINELEN]); Ll_str* cluster_names = nullptr; uint64_t tot_missing = 0; uintptr_t marker_ct = unfiltered_marker_ct - marker_exclude_ct; uintptr_t unfiltered_sample_ct4 = (unfiltered_sample_ct + 3) / 4; uintptr_t unfiltered_sample_ctl2 = QUATERCT_TO_WORDCT(unfiltered_sample_ct); uintptr_t sorted_sample_ctl = BITCT_TO_WORDCT(sorted_sample_ct); uintptr_t max_cluster_id_len = 0; uintptr_t possible_distinct_ct = 0; uintptr_t missing_cluster_ct = 0; uintptr_t y_start = 0; uintptr_t y_end = 0; uintptr_t line_idx = 0; int32_t y_code = chrom_info_ptr->xymt_codes[Y_OFFSET]; uint32_t y_present = ((y_code != -2) && is_set(chrom_info_ptr->chrom_mask, y_code)); int32_t retval = 0; Ll_str* ll_ptr; uintptr_t* loadbuf; uintptr_t* loadbuf_end; uintptr_t* cluster_zmask2s; uintptr_t* loadbuf_ptr; uintptr_t* cur_cluster_zmask2; uintptr_t* ulptr; uint32_t* cluster_sizes; uint32_t* marker_id_map; uint32_t* cluster_ref_cts; uint32_t* sample_lookup; char* cluster_ids; char* sorted_marker_ids; char* bufptr; char* bufptr2; int64_t* zc_entries; int64_t* zc_entries_end; uintptr_t cluster_ct; uintptr_t cluster_mct; // doubled if Y chrom present uintptr_t marker_uidx; uintptr_t ulii; uint64_t ullii; uint32_t sample_uidx; uint32_t cluster_idx; uint32_t slen; int32_t ii; if (y_present) { y_start = get_chrom_start_vidx(chrom_info_ptr, (uint32_t)y_code); y_end = get_chrom_end_vidx(chrom_info_ptr, (uint32_t)y_code); } if (bigstack_alloc_ul(unfiltered_sample_ctl2, &loadbuf)) { goto load_oblig_missing_ret_NOMEM; } loadbuf_end = &(loadbuf[unfiltered_sample_ctl2]); if (fopen_checked(om_ip->sample_fname, "r", &infile)) { goto load_oblig_missing_ret_OPEN_FAIL; } g_textbuf[MAXLINELEN - 1] = ' '; // two-pass load, same as load_clusters() // use loadbuf as duplicate IID detector fill_ulong_zero(sorted_sample_ctl, loadbuf); while (fgets(g_textbuf, MAXLINELEN, infile)) { line_idx++; if (!g_textbuf[MAXLINELEN - 1]) { LOGPREPRINTFWW("Error: Line %" PRIuPTR " of %s is pathologically long.\n", line_idx, om_ip->sample_fname); goto load_oblig_missing_ret_INVALID_FORMAT_2; } bufptr = skip_initial_spaces(g_textbuf); if (is_eoln_kns(*bufptr)) { continue; } if (bsearch_read_fam_indiv(bufptr, sorted_sample_ids, max_sample_id_len, sorted_sample_ct, &bufptr2, &ii, idbuf)) { LOGPREPRINTFWW("Error: Line %" PRIuPTR " of %s has fewer tokens than expected.\n", line_idx, om_ip->sample_fname); goto load_oblig_missing_ret_INVALID_FORMAT_2; } if (ii != -1) { if (is_set(loadbuf, ii)) { strchr(idbuf, '\t')[0] = ' '; LOGPREPRINTFWW("Error: Duplicate sample ID '%s' in %s.\n", idbuf, om_ip->sample_fname); goto load_oblig_missing_ret_INVALID_FORMAT_2; } set_bit(ii, loadbuf); slen = strlen_se(bufptr2); if (slen >= max_cluster_id_len) { max_cluster_id_len = slen + 1; } bufptr2[slen] = '\0'; if ((!cluster_names) || (strcmp(cluster_names->ss, bufptr2) && ((!cluster_names->next) || strcmp(cluster_names->next->ss, bufptr2)))) { if (bigstack_end_alloc_llstr(slen + 1, &ll_ptr)) { goto load_oblig_missing_ret_NOMEM; } ll_ptr->next = cluster_names; memcpy(ll_ptr->ss, bufptr2, slen + 1); cluster_names = ll_ptr; possible_distinct_ct++; } } } if (!feof(infile)) { goto load_oblig_missing_ret_READ_FAIL; } if (!max_cluster_id_len) { LOGERRPRINTFWW("Warning: --oblig-missing ignored, since no valid blocks were defined in %s.\n", om_ip->sample_fname); goto load_oblig_missing_ret_1; } if (bigstack_alloc_c(possible_distinct_ct * max_cluster_id_len, &cluster_ids)) { goto load_oblig_missing_ret_NOMEM; } for (ulii = 0; ulii < possible_distinct_ct; ulii++) { strcpy(&(cluster_ids[ulii * max_cluster_id_len]), cluster_names->ss); cluster_names = cluster_names->next; } bigstack_end_reset(bigstack_end_mark); qsort(cluster_ids, possible_distinct_ct, max_cluster_id_len, strcmp_casted); cluster_ct = collapse_duplicate_ids(cluster_ids, possible_distinct_ct, max_cluster_id_len, nullptr); bigstack_shrink_top(cluster_ids, cluster_ct * max_cluster_id_len); cluster_mct = cluster_ct * (y_present + 1); sample_lookup = (uint32_t*)malloc(unfiltered_sample_ct * sizeof(int32_t)); if (!sample_lookup) { goto load_oblig_missing_ret_NOMEM; } om_ip->sample_lookup = sample_lookup; if (bigstack_calloc_ui(cluster_mct, &cluster_sizes) || bigstack_calloc_ul(cluster_mct * unfiltered_sample_ctl2, &cluster_zmask2s)) { goto load_oblig_missing_ret_NOMEM; } fill_uint_one(unfiltered_sample_ct, sample_lookup); // second pass rewind(infile); while (fgets(g_textbuf, MAXLINELEN, infile)) { bufptr = skip_initial_spaces(g_textbuf); if (is_eoln_kns(*bufptr)) { continue; } bsearch_read_fam_indiv(bufptr, sorted_sample_ids, max_sample_id_len, sorted_sample_ct, &bufptr2, &ii, idbuf); if (ii == -1) { continue; } sample_uidx = sample_id_map[(uint32_t)ii]; slen = strlen_se(bufptr2); // guaranteed to succeed ii = bsearch_str(bufptr2, slen, cluster_ids, max_cluster_id_len, cluster_ct); set_bit(sample_uidx * 2, &(cluster_zmask2s[((uintptr_t)((uint32_t)ii)) * unfiltered_sample_ctl2])); cluster_sizes[(uint32_t)ii] += 1; sample_lookup[sample_uidx] = (uint32_t)ii; } if (fclose_null(&infile)) { goto load_oblig_missing_ret_READ_FAIL; } if (y_present) { init_quaterarr_from_bitarr(sex_male, unfiltered_sample_ct, loadbuf); cur_cluster_zmask2 = cluster_zmask2s; ulptr = &(cur_cluster_zmask2[cluster_ct * unfiltered_sample_ctl2]); for (cluster_idx = 0; cluster_idx < cluster_ct; cluster_idx++) { slen = 0; for (loadbuf_ptr = loadbuf; loadbuf_ptr < loadbuf_end;) { ulii = (*loadbuf_ptr++) & (*cur_cluster_zmask2++); slen += popcount2_long(ulii); *ulptr++ = ulii; } cluster_sizes[cluster_idx + cluster_ct] = slen; } } cluster_ref_cts = (uint32_t*)malloc(cluster_ct * 2 * sizeof(int32_t)); if (!cluster_ref_cts) { goto load_oblig_missing_ret_NOMEM; } om_ip->cluster_ref_cts = cluster_ref_cts; fill_uint_zero(cluster_ct * 2, cluster_ref_cts); retval = sort_item_ids(unfiltered_marker_ct, marker_exclude, marker_exclude_ct, marker_ids, max_marker_id_len, 0, 0, strcmp_deref, &sorted_marker_ids, &marker_id_map); if (retval) { goto load_oblig_missing_ret_1; } zc_entries = (int64_t*)g_bigstack_base; zc_entries_end = zc_entries; if (fopen_checked(om_ip->marker_fname, "r", &infile)) { goto load_oblig_missing_ret_OPEN_FAIL; } line_idx = 0; while (fgets(g_textbuf, MAXLINELEN, infile)) { line_idx++; if (!g_textbuf[MAXLINELEN - 1]) { LOGPREPRINTFWW("Error: Line %" PRIuPTR " of %s is pathologically long.\n", line_idx, om_ip->marker_fname); goto load_oblig_missing_ret_INVALID_FORMAT_2; } bufptr = skip_initial_spaces(g_textbuf); if (is_eoln_kns(*bufptr)) { continue; } bufptr2 = token_endnn(bufptr); ii = bsearch_str(bufptr, (uintptr_t)(bufptr2 - bufptr), sorted_marker_ids, max_marker_id_len, marker_ct); if (ii != -1) { marker_uidx = marker_id_map[(uint32_t)ii]; bufptr = skip_initial_spaces(bufptr2); if (is_eoln_kns(*bufptr)) { LOGPREPRINTFWW("Error: Line %" PRIuPTR " of %s has fewer tokens than expected.\n", line_idx, om_ip->marker_fname); goto load_oblig_missing_ret_INVALID_FORMAT_2; } slen = strlen_se(bufptr); ii = bsearch_str(bufptr, slen, cluster_ids, max_cluster_id_len, cluster_ct); if (ii != -1) { if (((unsigned char*)zc_entries_end) == g_bigstack_end) { goto load_oblig_missing_ret_NOMEM; } cluster_idx = (uint32_t)ii; if ((marker_uidx < y_end) && (marker_uidx >= y_start)) { cluster_idx += cluster_ct; } *zc_entries_end++ = (((uint64_t)marker_uidx) << 32) | ((uint64_t)cluster_idx); cluster_ref_cts[cluster_idx] += 1; } else { missing_cluster_ct++; } } } if (fclose_null(&infile)) { goto load_oblig_missing_ret_READ_FAIL; } if (missing_cluster_ct) { LOGERRPRINTFWW("Warning: %" PRIuPTR " entr%s in %s had block IDs missing from %s.\n", missing_cluster_ct, (missing_cluster_ct == 1)? "y" : "ies", om_ip->marker_fname, om_ip->sample_fname); } om_ip->entry_ct = (uintptr_t)(zc_entries_end - zc_entries); if (!om_ip->entry_ct) { LOGERRPRINTFWW("Warning: --oblig-missing ignored, since %s had no valid entries.\n", om_ip->marker_fname); goto load_oblig_missing_ret_1; } #ifdef __cplusplus std::sort(zc_entries, zc_entries_end); #else qsort(zc_entries, om_ip->entry_ct, sizeof(int64_t), llcmp); #endif om_ip->entries = (uint64_t*)malloc((om_ip->entry_ct + 1) * sizeof(int64_t)); if (!om_ip->entries) { goto load_oblig_missing_ret_NOMEM; } memcpy(om_ip->entries, zc_entries, om_ip->entry_ct * sizeof(int64_t)); om_ip->entries[om_ip->entry_ct] = ((uint64_t)unfiltered_marker_ct) << 32; om_ip->cluster_ct = cluster_ct; loadbuf[unfiltered_sample_ctl2 - 1] = 0; marker_uidx = unfiltered_marker_ct; // forces initial load do { ullii = *zc_entries++; if ((ullii >> 32) != marker_uidx) { marker_uidx = (ullii >> 32); if (fseeko(bedfile, bed_offset + ((uint64_t)marker_uidx) * unfiltered_sample_ct4, SEEK_SET)) { goto load_oblig_missing_ret_READ_FAIL; } if (load_raw(unfiltered_sample_ct4, bedfile, loadbuf)) { goto load_oblig_missing_ret_READ_FAIL; } // no need for het haploid handling here } loadbuf_ptr = loadbuf; cluster_idx = (uint32_t)ullii; cur_cluster_zmask2 = &(cluster_zmask2s[cluster_idx * unfiltered_sample_ctl2]); do { ulii = *cur_cluster_zmask2++; if (((*loadbuf_ptr) ^ ulii) & (ulii * 3)) { LOGPREPRINTFWW("Error: Nonmissing --oblig-missing genotype at variant '%s'. (To force it to missing, use --zero-cluster.)\n", &(marker_ids[marker_uidx * max_marker_id_len])); goto load_oblig_missing_ret_INVALID_FORMAT_2; } } while ((++loadbuf_ptr) < loadbuf_end); tot_missing += cluster_sizes[cluster_idx]; } while (zc_entries < zc_entries_end); LOGPRINTF("--oblig-missing: %" PRIu64 " call%s confirmed missing.\n", tot_missing, (tot_missing == 1)? "" : "s"); while (0) { load_oblig_missing_ret_NOMEM: retval = RET_NOMEM; break; load_oblig_missing_ret_OPEN_FAIL: retval = RET_OPEN_FAIL; break; load_oblig_missing_ret_READ_FAIL: retval = RET_READ_FAIL; break; load_oblig_missing_ret_INVALID_FORMAT_2: logerrprintb(); retval = RET_INVALID_FORMAT; break; } load_oblig_missing_ret_1: bigstack_double_reset(bigstack_mark, bigstack_end_mark); fclose_cond(infile); return retval; } int32_t filter_samples_file(char* filtername, char* sorted_sample_ids, uintptr_t sorted_ids_len, uintptr_t max_sample_id_len, uint32_t* id_map, uintptr_t unfiltered_sample_ct, uintptr_t* sample_exclude, uintptr_t* sample_exclude_ct_ptr, char* filtervals_flattened, uint32_t mfilter_col, uint32_t allow_no_samples) { FILE* infile = nullptr; unsigned char* bigstack_mark = g_bigstack_base; uintptr_t unfiltered_sample_ctl = BITCT_TO_WORDCT(unfiltered_sample_ct); uintptr_t include_ct = 0; uintptr_t max_filterval_len = 0; uintptr_t line_idx = 0; uint32_t filterval_ct = count_and_measure_multistr(filtervals_flattened, &max_filterval_len); int32_t retval = 0; char* sorted_filtervals; uintptr_t* sample_exclude_new; char* id_buf; char* bufptr; uint32_t filterval_idx; uint32_t slen; int32_t sample_idx; if (bigstack_alloc_c(max_sample_id_len, &id_buf) || bigstack_alloc_ul(unfiltered_sample_ctl, &sample_exclude_new) || bigstack_alloc_c(filterval_ct * max_filterval_len, &sorted_filtervals)) { goto filter_samples_file_ret_NOMEM; } fill_all_bits(unfiltered_sample_ct, sample_exclude_new); bufptr = filtervals_flattened; for (filterval_idx = 0; filterval_idx < filterval_ct; filterval_idx++) { slen = strlen(bufptr) + 1; memcpy(&(sorted_filtervals[filterval_idx * max_filterval_len]), bufptr, slen); bufptr = &(bufptr[slen]); } qsort(sorted_filtervals, filterval_ct, max_filterval_len, strcmp_casted); if (fopen_checked(filtername, "r", &infile)) { goto filter_samples_file_ret_OPEN_FAIL; } g_textbuf[MAXLINELEN - 1] = ' '; while (fgets(g_textbuf, MAXLINELEN, infile)) { line_idx++; if (!g_textbuf[MAXLINELEN - 1]) { snprintf(g_logbuf, LOGBUFLEN, "Error: Line %" PRIuPTR " of --filter file is pathologically long.\n", line_idx); goto filter_samples_file_ret_INVALID_FORMAT_2; } bufptr = skip_initial_spaces(g_textbuf); if (is_eoln_kns(*bufptr)) { continue; } if (bsearch_read_fam_indiv(bufptr, sorted_sample_ids, max_sample_id_len, sorted_ids_len, &bufptr, &sample_idx, id_buf)) { goto filter_samples_file_ret_MISSING_TOKENS; } if (sample_idx != -1) { sample_idx = id_map[(uint32_t)sample_idx]; if (!is_set(sample_exclude, sample_idx)) { if (mfilter_col > 1) { bufptr = next_token_mult(bufptr, mfilter_col - 1); } if (no_more_tokens_kns(bufptr)) { goto filter_samples_file_ret_MISSING_TOKENS; } if (bsearch_str(bufptr, strlen_se(bufptr), sorted_filtervals, max_filterval_len, filterval_ct) != -1) { if (is_set(sample_exclude_new, sample_idx)) { clear_bit(sample_idx, sample_exclude_new); include_ct++; } } } } } if (!feof(infile)) { goto filter_samples_file_ret_READ_FAIL; } if ((!include_ct) && (!allow_no_samples)) { LOGERRPRINTF("Error: All %s excluded by --filter.\n", g_species_plural); goto filter_samples_file_ret_ALL_SAMPLES_EXCLUDED; } LOGPRINTF("--filter: %" PRIuPTR " %s remaining.\n", include_ct, species_str(include_ct)); memcpy(sample_exclude, sample_exclude_new, unfiltered_sample_ctl * sizeof(intptr_t)); *sample_exclude_ct_ptr = unfiltered_sample_ct - include_ct; while (0) { filter_samples_file_ret_NOMEM: retval = RET_NOMEM; break; filter_samples_file_ret_OPEN_FAIL: retval = RET_OPEN_FAIL; break; filter_samples_file_ret_READ_FAIL: retval = RET_READ_FAIL; break; filter_samples_file_ret_MISSING_TOKENS: snprintf(g_logbuf, LOGBUFLEN, "Error: Line %" PRIuPTR " of --filter file has fewer tokens than expected.\n", line_idx); filter_samples_file_ret_INVALID_FORMAT_2: logerrprintb(); retval = RET_INVALID_FORMAT; break; filter_samples_file_ret_ALL_SAMPLES_EXCLUDED: retval = RET_ALL_SAMPLES_EXCLUDED; break; } bigstack_reset(bigstack_mark); fclose_cond(infile); return retval; } void filter_samples_bitfields(uintptr_t unfiltered_sample_ct, uintptr_t* sample_exclude, uintptr_t* sample_exclude_ct_ptr, uintptr_t* orfield, int32_t orfield_flip, uintptr_t* ornot) { // sample_exclude := sample_exclude | orfield | (~ornot) if !orfield_flip // := sample_exclude | (~orfield) | (~ornot) otherwise uintptr_t unfiltered_sample_ctl = BITCT_TO_WORDCT(unfiltered_sample_ct); uintptr_t* ieptr = sample_exclude; uintptr_t* ieend = &(sample_exclude[unfiltered_sample_ctl]); if (orfield_flip) { if (ornot) { do { *ieptr |= (~(*orfield++)) | (~(*ornot++)); } while (++ieptr < ieend); } else { do { *ieptr |= ~(*orfield++); } while (++ieptr < ieend); } } else { if (ornot) { do { *ieptr |= (*orfield++) | (~(*ornot++)); } while (++ieptr < ieend); } else { do { *ieptr |= *orfield++; } while (++ieptr < ieend); } } zero_trailing_bits(unfiltered_sample_ct, sample_exclude); *sample_exclude_ct_ptr = popcount_longs(sample_exclude, unfiltered_sample_ctl); } int32_t mind_filter(FILE* bedfile, uintptr_t bed_offset, char* outname, char* outname_end, double mind_thresh, uintptr_t unfiltered_marker_ct, uintptr_t* marker_exclude, uintptr_t marker_exclude_ct, uintptr_t unfiltered_sample_ct, uintptr_t* sample_exclude, uintptr_t* sample_exclude_ct_ptr, char* sample_ids, uintptr_t max_sample_id_len, uintptr_t* sex_male, Chrom_info* chrom_info_ptr, Oblig_missing_info* om_ip, uint32_t allow_no_samples) { uint32_t sample_exclude_ct = *sample_exclude_ct_ptr; uint32_t sample_ct = unfiltered_sample_ct - sample_exclude_ct; if (!sample_ct) { return 0; } unsigned char* bigstack_mark = g_bigstack_base; FILE* outfile = nullptr; uint32_t marker_ct = unfiltered_marker_ct - marker_exclude_ct; uintptr_t unfiltered_sample_ct4 = (unfiltered_sample_ct + 3) / 4; uintptr_t unfiltered_sample_ctl2 = QUATERCT_TO_WORDCT(unfiltered_sample_ct); uintptr_t unfiltered_sample_ctl = BITCT_TO_WORDCT(unfiltered_sample_ct); uintptr_t final_mask = get_final_mask(unfiltered_sample_ct); uintptr_t marker_idx = 0; uintptr_t y_start = 0; uintptr_t y_end = 0; uintptr_t* sample_male_include2 = nullptr; uint32_t unfiltered_sample_ctl2m1 = (unfiltered_sample_ct - 1) / BITCT2; uint32_t sample_uidx = 0; uint32_t sample_idx = 0; uint32_t removed_ct = 0; int32_t y_code = chrom_info_ptr->xymt_codes[Y_OFFSET]; uint32_t y_present = (y_code != -2) && is_set(chrom_info_ptr->chrom_mask, y_code); uint32_t nony_marker_ct = marker_ct; int32_t retval = 0; uint32_t mind_int_thresh[2]; uintptr_t* loadbuf; uintptr_t* newly_excluded; uintptr_t* lptr; uintptr_t* mptr; uint32_t* missing_cts; uint32_t* cluster_ref_cts; uint32_t* sample_lookup; uintptr_t marker_uidx; uint32_t cluster_ct; uint32_t cur_marker_ct; uint32_t sample_uidx_stop; uint32_t uii; uint32_t ujj; uintptr_t ulii; if (y_present) { y_start = get_chrom_start_vidx(chrom_info_ptr, (uint32_t)y_code); y_end = get_chrom_end_vidx(chrom_info_ptr, (uint32_t)y_code); if (bigstack_alloc_ul(unfiltered_sample_ctl2, &sample_male_include2)) { goto mind_filter_ret_NOMEM; } init_quaterarr_from_bitarr(sex_male, unfiltered_sample_ct, sample_male_include2); nony_marker_ct = marker_ct - (y_end - y_start - popcount_bit_idx(marker_exclude, y_start, y_end)); } if (bigstack_calloc_ui(unfiltered_sample_ct, &missing_cts) || bigstack_alloc_ul(unfiltered_sample_ctl2, &loadbuf) || bigstack_alloc_ul(unfiltered_sample_ctl, &newly_excluded)) { goto mind_filter_ret_NOMEM; } loadbuf[unfiltered_sample_ctl2 - 1] = 0; if (fseeko(bedfile, bed_offset, SEEK_SET)) { goto mind_filter_ret_READ_FAIL; } ujj = unfiltered_sample_ctl2 * BITCT2; marker_uidx = 0; for (; marker_idx < marker_ct; marker_uidx++, marker_idx++) { if (IS_SET(marker_exclude, marker_uidx)) { marker_uidx = next_unset_ul_unsafe(marker_exclude, marker_uidx); if (fseeko(bedfile, bed_offset + ((uint64_t)marker_uidx) * unfiltered_sample_ct4, SEEK_SET)) { goto mind_filter_ret_READ_FAIL; } } if (load_raw2(unfiltered_sample_ct4, unfiltered_sample_ctl2m1, final_mask, bedfile, loadbuf)) { goto mind_filter_ret_READ_FAIL; } // todo: switch to load_and_collapse() lptr = loadbuf; if ((marker_uidx >= y_end) || (marker_uidx < y_start)) { for (uii = 0; uii < ujj; uii += BITCT2) { ulii = *lptr++; ulii = (ulii & FIVEMASK) & ((~ulii) >> 1); // now ulii has single bit set only at missing positions while (ulii) { missing_cts[uii + CTZLU(ulii) / 2] += 1; ulii &= ulii - 1; } } } else { mptr = sample_male_include2; for (uii = 0; uii < ujj; uii += BITCT2) { ulii = *lptr++; ulii = (ulii & (*mptr++)) & ((~ulii) >> 1); while (ulii) { missing_cts[uii + CTZLU(ulii) / 2] += 1; ulii &= ulii - 1; } } } } fill_ulong_zero(unfiltered_sample_ctl, newly_excluded); if (!om_ip->entry_ct) { mind_int_thresh[0] = (int32_t)(mind_thresh * ((int32_t)nony_marker_ct) * (1 + SMALL_EPSILON)); mind_int_thresh[1] = (int32_t)(mind_thresh * ((int32_t)marker_ct) * (1 + SMALL_EPSILON)); do { sample_uidx = next_unset_unsafe(sample_exclude, sample_uidx); sample_uidx_stop = next_set(sample_exclude, sample_uidx, unfiltered_sample_ct); sample_idx += sample_uidx_stop - sample_uidx; do { if (missing_cts[sample_uidx] > mind_int_thresh[is_set(sex_male, sample_uidx)]) { SET_BIT(sample_uidx, newly_excluded); removed_ct++; } } while (++sample_uidx < sample_uidx_stop); } while (sample_idx < sample_ct); } else { sample_lookup = om_ip->sample_lookup; cluster_ref_cts = om_ip->cluster_ref_cts; cluster_ct = om_ip->cluster_ct; do { sample_uidx = next_unset_unsafe(sample_exclude, sample_uidx); sample_uidx_stop = next_set(sample_exclude, sample_uidx, unfiltered_sample_ct); sample_idx += sample_uidx_stop - sample_uidx; do { uii = sample_lookup[sample_uidx]; ujj = 0; if (is_set(sex_male, sample_uidx)) { cur_marker_ct = marker_ct; if (uii != 0xffffffffU) { ujj = cluster_ref_cts[uii] + cluster_ref_cts[uii + cluster_ct]; } } else { cur_marker_ct = nony_marker_ct; if (uii != 0xffffffffU) { ujj = cluster_ref_cts[uii]; } } if ((missing_cts[sample_uidx] - ujj) > (uint32_t)((int32_t)(mind_thresh * ((int32_t)(cur_marker_ct - ujj)) * (1 + SMALL_EPSILON)))) { SET_BIT(sample_uidx, newly_excluded); removed_ct++; } } while (++sample_uidx < sample_uidx_stop); } while (sample_idx < sample_ct); } if (removed_ct) { bitvec_or(newly_excluded, unfiltered_sample_ctl, sample_exclude); memcpy(outname_end, ".irem", 6); if (fopen_checked(outname, "w", &outfile)) { goto mind_filter_ret_OPEN_FAIL; } sample_uidx = 0; for (sample_idx = 0; sample_idx < removed_ct; sample_idx++, sample_uidx++) { next_set_unsafe_ck(newly_excluded, &sample_uidx); fputs(&(sample_ids[sample_uidx * max_sample_id_len]), outfile); putc_unlocked('\n', outfile); } if (fclose_null(&outfile)) { goto mind_filter_ret_WRITE_FAIL; } } *sample_exclude_ct_ptr += removed_ct; if ((*sample_exclude_ct_ptr == unfiltered_sample_ct) && (!allow_no_samples)) { LOGERRPRINTF("Error: All %s removed due to missing genotype data (--mind).\n", g_species_plural); LOGPRINTFWW("IDs written to %s .\n", outname); goto mind_filter_ret_ALL_SAMPLES_EXCLUDED; } LOGPRINTF("%u %s removed due to missing genotype data (--mind).\n", removed_ct, species_str(removed_ct)); if (removed_ct) { LOGPRINTFWW("ID%s written to %s .\n", (removed_ct == 1)? "" : "s", outname); } while (0) { mind_filter_ret_NOMEM: retval = RET_NOMEM; break; mind_filter_ret_OPEN_FAIL: retval = RET_OPEN_FAIL; break; mind_filter_ret_READ_FAIL: retval = RET_READ_FAIL; break; mind_filter_ret_WRITE_FAIL: retval = RET_WRITE_FAIL; break; mind_filter_ret_ALL_SAMPLES_EXCLUDED: retval = RET_ALL_SAMPLES_EXCLUDED; break; } bigstack_reset(bigstack_mark); fclose_cond(outfile); return retval; } static inline void single_marker_freqs_and_hwe(uintptr_t unfiltered_sample_ctl2, uintptr_t* lptr, uintptr_t* sample_include2, uintptr_t* founder_include2, uintptr_t* founder_ctrl_include2, uintptr_t* founder_case_include2, uintptr_t sample_ct, uint32_t* ll_ctp, uint32_t* lh_ctp, uint32_t* hh_ctp, uint32_t sample_f_ct, uint32_t* ll_ctfp, uint32_t* lh_ctfp, uint32_t* hh_ctfp, uint32_t hwe_or_geno_needed, uintptr_t sample_f_ctrl_ct, uint32_t* ll_hwep, uint32_t* lh_hwep, uint32_t* hh_hwep, int32_t hardy_needed, uintptr_t sample_f_case_ct, uint32_t* ll_case_hwep, uint32_t* lh_case_hwep, uint32_t* hh_case_hwep) { // This is best understood from the bottom third up (which is the order it // was written). It's way overkill for just determining genotype // frequencies, but a ruthlessly optimized version is needed for e.g. // permutation testing so we may as well get it working here. // // The idea, which underlies the IBS and LD pruners as well, is to obtain // multiple popcounts at once. In the case of PLINK frequency/HWE // evaluation, we need 6-9 numbers: // homozyg minor, het, homozyg major counts for all samples // homozyg minor, het, homozyg major counts for all founders // sometimes, homozyg minor, het, homozyg major counts for all ctrl founders // // Given a buffer with PLINK binary genotypes for a single marker, let // A := genotype & 0x5555... // B := (genotype >> 1) & 0x5555... // C := A & B // Then, // popcount(C) = homozyg major ct // popcount(B) = het ct + homozyg major ct // popcount(A) = missing_ct + homozyg major ct // = sample_ct - homozyg minor ct - het ct // // Thus, with the appropriate sample_ct and these three popcounts, we can // calculate a set of genotype counts. We perform the // not-that-exploitative version of these calculations in the bottom third of // this function, to deal with the remainder that doesn't fit into the // 12-word block size of the main loops. // // The middle third is a 12-word block popcount for 32-bit platforms (see // popcount_longs() in plink_common.c; this is 12 words instead of 6 since // odd bits of the popcount targets are guaranteed to be zero, delaying // overflow). It could be improved a bit, but we care more about reliability // than blistering speed for the 32-bit build (since it's used to check the // results of the actually blistering 64-bit code...). Sure, hardware // popcount is significantly faster, but most machines running 32-bit OSes // don't have it. // // The top third is the portable Lauradoux/Walisch loop. uint32_t tot_a = 0; uint32_t tot_b = 0; uint32_t tot_c = 0; uint32_t tot_a_f = 0; uint32_t tot_b_f = 0; uint32_t tot_c_f = 0; uint32_t tot_a_hwe = 0; uint32_t tot_b_hwe = 0; uint32_t tot_c_hwe = 0; uint32_t tot_a_chwe = 0; uint32_t tot_b_chwe = 0; uint32_t tot_c_chwe = 0; uintptr_t* lptr_end = &(lptr[unfiltered_sample_ctl2]); uintptr_t loader; uintptr_t loader2; uintptr_t loader3; #ifdef __LP64__ uintptr_t cur_decr = 120; uintptr_t* lptr_12x_end; unfiltered_sample_ctl2 -= unfiltered_sample_ctl2 % 12; while (unfiltered_sample_ctl2 >= 120) { single_marker_freqs_and_hwe_loop: lptr_12x_end = &(lptr[cur_decr]); count_3freq_1920b((__m128i*)lptr, (__m128i*)lptr_12x_end, (__m128i*)sample_include2, &tot_a, &tot_b, &tot_c); count_3freq_1920b((__m128i*)lptr, (__m128i*)lptr_12x_end, (__m128i*)founder_include2, &tot_a_f, &tot_b_f, &tot_c_f); if (hwe_or_geno_needed) { count_3freq_1920b((__m128i*)lptr, (__m128i*)lptr_12x_end, (__m128i*)founder_ctrl_include2, &tot_a_hwe, &tot_b_hwe, &tot_c_hwe); founder_ctrl_include2 = &(founder_ctrl_include2[cur_decr]); if (hardy_needed) { count_3freq_1920b((__m128i*)lptr, (__m128i*)lptr_12x_end, (__m128i*)founder_case_include2, &tot_a_chwe, &tot_b_chwe, &tot_c_chwe); founder_case_include2 = &(founder_case_include2[cur_decr]); } } lptr = lptr_12x_end; sample_include2 = &(sample_include2[cur_decr]); founder_include2 = &(founder_include2[cur_decr]); unfiltered_sample_ctl2 -= cur_decr; } if (unfiltered_sample_ctl2) { cur_decr = unfiltered_sample_ctl2; goto single_marker_freqs_and_hwe_loop; } #else uintptr_t* lptr_twelve_end = &(lptr[unfiltered_sample_ctl2 - unfiltered_sample_ctl2 % 12]); while (lptr < lptr_twelve_end) { count_3freq_48b(lptr, sample_include2, &tot_a, &tot_b, &tot_c); count_3freq_48b(lptr, founder_include2, &tot_a_f, &tot_b_f, &tot_c_f); if (hwe_or_geno_needed) { count_3freq_48b(lptr, founder_ctrl_include2, &tot_a_hwe, &tot_b_hwe, &tot_c_hwe); founder_ctrl_include2 = &(founder_ctrl_include2[12]); if (hardy_needed) { count_3freq_48b(lptr, founder_case_include2, &tot_a_chwe, &tot_b_chwe, &tot_c_chwe); founder_case_include2 = &(founder_case_include2[12]); } } lptr = &(lptr[12]); sample_include2 = &(sample_include2[12]); founder_include2 = &(founder_include2[12]); } #endif while (lptr < lptr_end) { loader = *lptr++; loader2 = *sample_include2++; loader3 = (loader >> 1) & loader2; loader2 &= loader; // N.B. because of the construction of sample_include2, only even-numbered // bits can be present here. So popcount2_long is safe. tot_a += popcount2_long(loader2); tot_b += popcount2_long(loader3); tot_c += popcount2_long(loader & loader3); loader2 = *founder_include2++; loader3 = (loader >> 1) & loader2; loader2 &= loader; tot_a_f += popcount2_long(loader2); tot_b_f += popcount2_long(loader3); tot_c_f += popcount2_long(loader & loader3); if (hwe_or_geno_needed) { loader2 = *founder_ctrl_include2++; loader3 = (loader >> 1) & loader2; loader2 &= loader; tot_a_hwe += popcount2_long(loader2); tot_b_hwe += popcount2_long(loader3); tot_c_hwe += popcount2_long(loader & loader3); if (hardy_needed) { loader2 = *founder_case_include2++; loader3 = (loader >> 1) & loader2; loader2 &= loader; tot_a_chwe += popcount2_long(loader2); tot_b_chwe += popcount2_long(loader3); tot_c_chwe += popcount2_long(loader & loader3); } } } *hh_ctp = tot_c; *lh_ctp = tot_b - tot_c; *ll_ctp = sample_ct - tot_a - *lh_ctp; *hh_ctfp = tot_c_f; *lh_ctfp = tot_b_f - tot_c_f; *ll_ctfp = sample_f_ct - tot_a_f - *lh_ctfp; if (hwe_or_geno_needed) { *hh_hwep = tot_c_hwe; *lh_hwep = tot_b_hwe - tot_c_hwe; *ll_hwep = sample_f_ctrl_ct - tot_a_hwe - *lh_hwep; if (hardy_needed) { *hh_case_hwep = tot_c_chwe; *lh_case_hwep = tot_b_chwe - tot_c_chwe; *ll_case_hwep = sample_f_case_ct - tot_a_chwe - *lh_case_hwep; } } } static inline uint32_t nonmissing_present_diff(uintptr_t unfiltered_sample_ctl2, uintptr_t* lptr, uintptr_t* sample_include2, uintptr_t* sample_male_include2) { // possible todo: write entries to .ynm file, using same format as .hh uintptr_t* lptr_end = &(lptr[unfiltered_sample_ctl2]); uintptr_t loader; uintptr_t loader2; do { loader = *lptr++; loader2 = (*sample_include2++) & (~(*sample_male_include2++)); // when really bored, check if compiler translates this into andnot // operations if ((~((~(loader >> 1)) & loader)) & loader2) { return 1; } } while (lptr < lptr_end); return 0; } int32_t calc_freqs_and_hwe(FILE* bedfile, char* outname, char* outname_end, uintptr_t unfiltered_marker_ct, uintptr_t* marker_exclude, uintptr_t marker_ct, char* marker_ids, uintptr_t max_marker_id_len, uintptr_t unfiltered_sample_ct, uintptr_t* sample_exclude, uintptr_t sample_exclude_ct, char* sample_ids, uintptr_t max_sample_id_len, uintptr_t* founder_info, int32_t nonfounders, int32_t maf_succ, double* set_allele_freqs, uintptr_t bed_offset, uint32_t hwe_needed, uint32_t hwe_all, uint32_t hardy_needed, uint32_t min_ac, uint32_t max_ac, double geno_thresh, uintptr_t* pheno_nm, uintptr_t* pheno_c, int32_t** hwe_lls_ptr, int32_t** hwe_lhs_ptr, int32_t** hwe_hhs_ptr, int32_t** hwe_ll_cases_ptr, int32_t** hwe_lh_cases_ptr, int32_t** hwe_hh_cases_ptr, int32_t** hwe_ll_allfs_ptr, int32_t** hwe_lh_allfs_ptr, int32_t** hwe_hh_allfs_ptr, int32_t** hwe_hapl_allfs_ptr, int32_t** hwe_haph_allfs_ptr, uintptr_t** geno_excl_bitfield_ptr, uintptr_t** ac_excl_bitfield_ptr, uint32_t* sample_male_ct_ptr, uint32_t* sample_f_ct_ptr, uint32_t* sample_f_male_ct_ptr, Chrom_info* chrom_info_ptr, Oblig_missing_info* om_ip, uintptr_t* sex_nm, uintptr_t* sex_male, uint32_t is_split_chrom, uint32_t* hh_exists_ptr) { FILE* hhfile = nullptr; uintptr_t unfiltered_sample_ct4 = (unfiltered_sample_ct + 3) / 4; uintptr_t unfiltered_sample_ctl = BITCT_TO_WORDCT(unfiltered_sample_ct); uintptr_t unfiltered_sample_ctv2 = 2 * unfiltered_sample_ctl; uintptr_t unfiltered_marker_ctl = BITCT_TO_WORDCT(unfiltered_marker_ct); int32_t retval = 0; uint32_t pct = 1; uint32_t sample_ct = unfiltered_sample_ct - sample_exclude_ct; double sample_ct_recip = 1.0 / ((double)((int32_t)sample_ct)); // sum of nonmissing rates over all markers // rate is in [0, 1] for each marker, so sum is in [0, marker_ct]. double nonmissing_rate_tot = 0.0; // track this to defend against Y chromosome/0 males pathological case uintptr_t nonmissing_rate_tot_max = marker_ct; uint32_t sample_f_ct = sample_ct; uintptr_t sample_f_ctrl_ct = sample_ct; uintptr_t sample_f_case_ct = sample_ct; unsigned char* bigstack_mark = g_bigstack_base; uint32_t ll_hwe = 0; uint32_t lh_hwe = 0; uint32_t hh_hwe = 0; uint32_t ll_case_hwe = 0; uint32_t lh_case_hwe = 0; uint32_t hh_case_hwe = 0; uint32_t cur_chrom_idx = 0; uint32_t nonmissing_nonmale_y = 0; int32_t ii = chrom_info_ptr->chrom_file_order[0]; uint32_t is_haploid = is_set(chrom_info_ptr->haploid_mask, ii); uint32_t next_chrom_start = chrom_info_ptr->chrom_fo_vidx_start[1]; uint32_t is_x = (ii == chrom_info_ptr->xymt_codes[X_OFFSET]); uint32_t is_y = (ii == chrom_info_ptr->xymt_codes[Y_OFFSET]); uint32_t ll_ct = 0; uint32_t lh_ct = 0; uint32_t hh_ct = 0; uint32_t ll_ctf = 0; uint32_t lh_ctf = 0; uint32_t hh_ctf = 0; uint32_t ukk = 0; uint32_t cur_oblig_missing = 0; uint32_t om_cluster_ct = 0; uint32_t* om_cluster_sizes = nullptr; int32_t* hwe_lls = nullptr; int32_t* hwe_lhs = nullptr; int32_t* hwe_hhs = nullptr; int32_t* hwe_ll_cases = nullptr; int32_t* hwe_lh_cases = nullptr; int32_t* hwe_hh_cases = nullptr; int32_t* hwe_ll_allfs = nullptr; int32_t* hwe_lh_allfs = nullptr; int32_t* hwe_hh_allfs = nullptr; uintptr_t* sample_nonmale_include2 = nullptr; uintptr_t* sample_male_include2 = nullptr; uintptr_t* founder_nonmale_include2 = nullptr; uintptr_t* founder_ctrl_nonmale_include2 = nullptr; uintptr_t* founder_male_include2 = nullptr; uintptr_t* founder_case_include2 = nullptr; uintptr_t* founder_case_nonmale_include2 = nullptr; uintptr_t* geno_excl_bitfield = nullptr; uintptr_t* ac_excl_bitfield = nullptr; uint64_t* om_entry_ptr = nullptr; uint32_t sample_nonmale_ct = 0; uint32_t sample_f_nonmale_ct = 0; uint32_t sample_f_ctl_nonmale_ct = 0; uint32_t sample_f_case_nonmale_ct = 0; uint64_t hethap_ct = 0; uint64_t cur_om_entry = 0; double male_ct_recip = 0; uint32_t* om_sample_lookup; int32_t* hwe_hapl_allfs; int32_t* hwe_haph_allfs; uintptr_t* loadbuf; uintptr_t* sample_include2; uintptr_t* founder_include2; uintptr_t* founder_ctrl_include2; uintptr_t* tmp_sample_excl_mask; uintptr_t* tmp_sample_excl_mask2; uintptr_t loop_end; uintptr_t marker_uidx; uintptr_t marker_idx; uintptr_t sample_uidx; uintptr_t sample_idx; uintptr_t ulii; uint32_t sample_male_ct; uint32_t sample_f_male_ct; uint32_t nonmales_needed; uint32_t males_needed; uint32_t uii; uint32_t ujj; double maf; double cur_genotyping_rate; if (!hwe_needed) { *hwe_lls_ptr = (int32_t*)g_bigstack_base; } else { if (bigstack_alloc_i(unfiltered_marker_ct, &hwe_lls) || bigstack_alloc_i(unfiltered_marker_ct, &hwe_lhs) || bigstack_alloc_i(unfiltered_marker_ct, &hwe_hhs)) { goto calc_freqs_and_hwe_ret_NOMEM; } *hwe_lls_ptr = hwe_lls; *hwe_lhs_ptr = hwe_lhs; *hwe_hhs_ptr = hwe_hhs; if (hardy_needed) { if (bigstack_alloc_i(unfiltered_marker_ct, &hwe_ll_cases) || bigstack_alloc_i(unfiltered_marker_ct, &hwe_lh_cases) || bigstack_alloc_i(unfiltered_marker_ct, &hwe_hh_cases)) { goto calc_freqs_and_hwe_ret_NOMEM; } } *hwe_ll_cases_ptr = hwe_ll_cases; *hwe_lh_cases_ptr = hwe_lh_cases; *hwe_hh_cases_ptr = hwe_hh_cases; } if (bigstack_calloc_i(unfiltered_marker_ct, &hwe_ll_allfs) || bigstack_calloc_i(unfiltered_marker_ct, &hwe_lh_allfs) || bigstack_calloc_i(unfiltered_marker_ct, &hwe_hh_allfs) || bigstack_calloc_i(unfiltered_marker_ct, &hwe_hapl_allfs) || bigstack_calloc_i(unfiltered_marker_ct, &hwe_haph_allfs)) { goto calc_freqs_and_hwe_ret_NOMEM; } *hwe_ll_allfs_ptr = hwe_ll_allfs; *hwe_lh_allfs_ptr = hwe_lh_allfs; *hwe_hh_allfs_ptr = hwe_hh_allfs; *hwe_hapl_allfs_ptr = hwe_hapl_allfs; *hwe_haph_allfs_ptr = hwe_haph_allfs; if ((!pheno_c) || is_split_chrom) { hwe_all = 1; } if (geno_thresh < 1.0) { if (bigstack_calloc_ul(unfiltered_marker_ctl, geno_excl_bitfield_ptr)) { goto calc_freqs_and_hwe_ret_NOMEM; } geno_excl_bitfield = *geno_excl_bitfield_ptr; // change this to a minimum nonmissing rate geno_thresh = (1.0 - geno_thresh) * (1 - SMALL_EPSILON); } if ((min_ac > 0) || (max_ac < sample_ct)) { if (bigstack_calloc_ul(unfiltered_marker_ctl, ac_excl_bitfield_ptr)) { goto calc_freqs_and_hwe_ret_NOMEM; } ac_excl_bitfield = *ac_excl_bitfield_ptr; } bigstack_mark = g_bigstack_base; if (bigstack_alloc_ul(unfiltered_sample_ctv2, &loadbuf) || bigstack_alloc_ul(unfiltered_sample_ctv2, &sample_include2)) { goto calc_freqs_and_hwe_ret_NOMEM; } loadbuf[unfiltered_sample_ctv2 - 2] = 0; loadbuf[unfiltered_sample_ctv2 - 1] = 0; init_quaterarr_from_inverted_bitarr(sample_exclude, unfiltered_sample_ct, sample_include2); ii = chrom_info_ptr->xymt_codes[X_OFFSET]; nonmales_needed = (!is_split_chrom) && (ii != -2) && is_set(chrom_info_ptr->chrom_mask, ii); ii = chrom_info_ptr->xymt_codes[Y_OFFSET]; males_needed = nonmales_needed || ((!is_split_chrom) && (ii != -2) && is_set(chrom_info_ptr->chrom_mask, ii)); if (bigstack_alloc_ul(unfiltered_sample_ctv2, &sample_male_include2)) { goto calc_freqs_and_hwe_ret_NOMEM; } memcpy(sample_male_include2, sample_include2, unfiltered_sample_ctv2 * sizeof(intptr_t)); apply_bitarr_mask_to_quaterarr_01(sex_male, unfiltered_sample_ct, sample_male_include2); sample_male_ct = popcount01_longs(sample_male_include2, unfiltered_sample_ctv2); if (sample_male_ct) { male_ct_recip = 1.0 / ((double)((int32_t)sample_male_ct)); } *sample_male_ct_ptr = sample_male_ct; sample_f_male_ct = sample_male_ct; if (males_needed) { founder_male_include2 = sample_male_include2; if (nonmales_needed) { if (bigstack_alloc_ul(unfiltered_sample_ctv2, &sample_nonmale_include2)) { goto calc_freqs_and_hwe_ret_NOMEM; } memcpy(sample_nonmale_include2, sample_include2, unfiltered_sample_ctv2 * sizeof(intptr_t)); apply_excl_intersect_to_quaterarr_01(sex_nm, sex_male, unfiltered_sample_ct, sample_nonmale_include2); sample_nonmale_ct = popcount01_longs(sample_nonmale_include2, unfiltered_sample_ctv2); sample_f_nonmale_ct = sample_nonmale_ct; founder_nonmale_include2 = sample_nonmale_include2; founder_ctrl_nonmale_include2 = sample_nonmale_include2; } } founder_include2 = sample_include2; if (bigstack_alloc_ul(unfiltered_sample_ctl, &tmp_sample_excl_mask)) { goto calc_freqs_and_hwe_ret_NOMEM; } memcpy(tmp_sample_excl_mask, sample_exclude, unfiltered_sample_ctl * sizeof(intptr_t)); if (!nonfounders) { if (bigstack_alloc_ul(unfiltered_sample_ctv2, &founder_include2)) { goto calc_freqs_and_hwe_ret_NOMEM; } bitvec_ornot(founder_info, unfiltered_sample_ctl, tmp_sample_excl_mask); zero_trailing_bits(unfiltered_sample_ct, tmp_sample_excl_mask); init_quaterarr_from_inverted_bitarr(tmp_sample_excl_mask, unfiltered_sample_ct, founder_include2); if (males_needed) { if (bigstack_alloc_ul(unfiltered_sample_ctv2, &founder_male_include2)) { goto calc_freqs_and_hwe_ret_NOMEM; } memcpy(founder_male_include2, sample_male_include2, unfiltered_sample_ctl * 2 * sizeof(intptr_t)); apply_bitarr_mask_to_quaterarr_01(founder_info, unfiltered_sample_ct, founder_male_include2); sample_f_male_ct = popcount01_longs(founder_male_include2, unfiltered_sample_ctv2); if (nonmales_needed) { if (bigstack_alloc_ul(unfiltered_sample_ctv2, &founder_nonmale_include2)) { goto calc_freqs_and_hwe_ret_NOMEM; } memcpy(founder_nonmale_include2, sample_nonmale_include2, unfiltered_sample_ctv2 * sizeof(intptr_t)); apply_bitarr_mask_to_quaterarr_01(founder_info, unfiltered_sample_ct, founder_nonmale_include2); sample_f_nonmale_ct = popcount01_longs(founder_nonmale_include2, unfiltered_sample_ctv2); } } founder_ctrl_include2 = founder_include2; sample_f_ct = popcount_longs_exclude(founder_info, sample_exclude, unfiltered_sample_ctl); sample_f_ctrl_ct = sample_f_ct; } else { founder_ctrl_include2 = founder_include2; } // bugfix: this previously failed to initialize founder_ctrl_include2 and // founder_case_include2 properly if --hardy was used in a situation where // hwe_all would be set (e.g. all-case datasets). if ((!hwe_all) || hardy_needed) { if (bigstack_alloc_ul(unfiltered_sample_ctv2, &founder_ctrl_include2) || bigstack_alloc_ul(unfiltered_sample_ctl, &tmp_sample_excl_mask2)) { goto calc_freqs_and_hwe_ret_NOMEM; } memcpy(tmp_sample_excl_mask2, tmp_sample_excl_mask, unfiltered_sample_ctl * sizeof(intptr_t)); bitvec_ornot(pheno_nm, unfiltered_sample_ctl, tmp_sample_excl_mask2); bitvec_or(pheno_c, unfiltered_sample_ctl, tmp_sample_excl_mask2); zero_trailing_bits(unfiltered_sample_ct, tmp_sample_excl_mask2); // tmp_sample_excl_mask2 is now set for each sample who is excluded, or a // nonfounder, or is noncontrol. sample_f_ctrl_ct = unfiltered_sample_ct - popcount_longs(tmp_sample_excl_mask2, unfiltered_sample_ctl); init_quaterarr_from_inverted_bitarr(tmp_sample_excl_mask2, unfiltered_sample_ct, founder_ctrl_include2); if (nonmales_needed) { if (bigstack_alloc_ul(unfiltered_sample_ctv2, &founder_ctrl_nonmale_include2)) { goto calc_freqs_and_hwe_ret_NOMEM; } memcpy(founder_ctrl_nonmale_include2, sample_nonmale_include2, unfiltered_sample_ctv2 * sizeof(intptr_t)); apply_bitarr_excl_to_quaterarr_01(tmp_sample_excl_mask2, unfiltered_sample_ct, founder_ctrl_nonmale_include2); sample_f_ctl_nonmale_ct = popcount01_longs(founder_ctrl_nonmale_include2, unfiltered_sample_ctv2); } if (hardy_needed) { if (bigstack_alloc_ul(unfiltered_sample_ctv2, &founder_case_include2)) { goto calc_freqs_and_hwe_ret_NOMEM; } bitvec_ornot(pheno_nm, unfiltered_sample_ctl, tmp_sample_excl_mask); bitvec_ornot(pheno_c, unfiltered_sample_ctl, tmp_sample_excl_mask); zero_trailing_bits(unfiltered_sample_ct, tmp_sample_excl_mask); sample_f_case_ct = unfiltered_sample_ct - popcount_longs(tmp_sample_excl_mask, unfiltered_sample_ctl); init_quaterarr_from_inverted_bitarr(tmp_sample_excl_mask, unfiltered_sample_ct, founder_case_include2); if (nonmales_needed) { if (bigstack_alloc_ul(unfiltered_sample_ctv2, &founder_case_nonmale_include2)) { goto calc_freqs_and_hwe_ret_NOMEM; } memcpy(founder_case_nonmale_include2, sample_nonmale_include2, unfiltered_sample_ctv2 * sizeof(intptr_t)); apply_bitarr_excl_to_quaterarr_01(tmp_sample_excl_mask, unfiltered_sample_ct, founder_case_nonmale_include2); sample_f_case_nonmale_ct = popcount01_longs(founder_case_nonmale_include2, unfiltered_sample_ctv2); } } } if (om_ip->entry_ct) { om_entry_ptr = om_ip->entries; om_cluster_ct = om_ip->cluster_ct; om_sample_lookup = om_ip->sample_lookup; cur_om_entry = *om_entry_ptr; if (bigstack_calloc_ui(om_cluster_ct * 2, &om_cluster_sizes)) { goto calc_freqs_and_hwe_ret_NOMEM; } sample_uidx = 0; for (sample_idx = 0; sample_idx < sample_ct; sample_uidx++, sample_idx++) { next_unset_ul_unsafe_ck(sample_exclude, &sample_uidx); uii = om_sample_lookup[sample_uidx]; if (uii != 0xffffffffU) { om_cluster_sizes[uii] += 1; if (is_set(sex_male, sample_uidx)) { om_cluster_sizes[uii + om_cluster_ct] += 1; } } } } *sample_f_ct_ptr = sample_f_ct; *sample_f_male_ct_ptr = sample_f_male_ct; if (fseeko(bedfile, bed_offset, SEEK_SET)) { goto calc_freqs_and_hwe_ret_READ_FAIL; } marker_uidx = 0; marker_idx = 0; logprint("Calculating allele frequencies..."); fputs(" 0%", stdout); fflush(stdout); if (is_split_chrom) { // only set is_haploid if all chromosomes are haploid is_haploid = (chrom_info_ptr->haploid_mask[0]) & 1; is_x = 0; is_y = 0; next_chrom_start = unfiltered_marker_ct; } for (; pct <= 100; pct++) { loop_end = ((uint64_t)pct * marker_ct) / 100LU; for (; marker_idx < loop_end; marker_uidx++, marker_idx++) { if (IS_SET(marker_exclude, marker_uidx)) { marker_uidx = next_unset_ul_unsafe(marker_exclude, marker_uidx); if (fseeko(bedfile, bed_offset + ((uint64_t)marker_uidx) * unfiltered_sample_ct4, SEEK_SET)) { goto calc_freqs_and_hwe_ret_READ_FAIL; } } if (load_raw(unfiltered_sample_ct4, bedfile, loadbuf)) { goto calc_freqs_and_hwe_ret_READ_FAIL; } if (marker_uidx >= next_chrom_start) { do { next_chrom_start = chrom_info_ptr->chrom_fo_vidx_start[(++cur_chrom_idx) + 1]; } while (marker_uidx >= next_chrom_start); ii = chrom_info_ptr->chrom_file_order[cur_chrom_idx]; is_haploid = is_set(chrom_info_ptr->haploid_mask, ii); is_x = (ii == chrom_info_ptr->xymt_codes[X_OFFSET]); is_y = (ii == chrom_info_ptr->xymt_codes[Y_OFFSET]); } if (om_entry_ptr) { cur_oblig_missing = 0; while ((cur_om_entry >> 32) < marker_uidx) { cur_om_entry = *(++om_entry_ptr); } while ((cur_om_entry >> 32) == marker_uidx) { cur_oblig_missing += om_cluster_sizes[(uint32_t)cur_om_entry]; cur_om_entry = *(++om_entry_ptr); } } if (!is_haploid) { single_marker_freqs_and_hwe(unfiltered_sample_ctv2, loadbuf, sample_include2, founder_include2, founder_ctrl_include2, founder_case_include2, sample_ct, &ll_ct, &lh_ct, &hh_ct, sample_f_ct, &ll_ctf, &lh_ctf, &hh_ctf, hwe_needed, sample_f_ctrl_ct, &ll_hwe, &lh_hwe, &hh_hwe, hardy_needed, sample_f_case_ct, &ll_case_hwe, &lh_case_hwe, &hh_case_hwe); hwe_ll_allfs[marker_uidx] = ll_ctf; hwe_lh_allfs[marker_uidx] = lh_ctf; hwe_hh_allfs[marker_uidx] = hh_ctf; uii = ll_ct + lh_ct + hh_ct; if (!cur_oblig_missing) { cur_genotyping_rate = ((int32_t)uii) * sample_ct_recip; } else { if (sample_ct - cur_oblig_missing) { cur_genotyping_rate = ((int32_t)uii) / ((double)((int32_t)(sample_ct - cur_oblig_missing))); } else { cur_genotyping_rate = 0; nonmissing_rate_tot_max -= 1; } } if (ac_excl_bitfield) { if (ll_ctf < hh_ctf) { uii = 2 * ll_ctf + lh_ctf; } else { uii = 2 * hh_ctf + lh_ctf; } if ((uii < min_ac) || (uii > max_ac)) { set_bit(marker_uidx, ac_excl_bitfield); } } uii = 2 * (ll_ctf + lh_ctf + hh_ctf + maf_succ); if (!uii) { // avoid 0/0 division set_allele_freqs[marker_uidx] = 0.5; } else { set_allele_freqs[marker_uidx] = ((double)(2 * hh_ctf + lh_ctf + maf_succ)) / ((double)uii); } if (hwe_needed) { hwe_lls[marker_uidx] = ll_hwe; hwe_lhs[marker_uidx] = lh_hwe; hwe_hhs[marker_uidx] = hh_hwe; if (hardy_needed) { hwe_ll_cases[marker_uidx] = ll_case_hwe; hwe_lh_cases[marker_uidx] = lh_case_hwe; hwe_hh_cases[marker_uidx] = hh_case_hwe; } } } else { uii = 0; ujj = 0; if (is_x || is_y) { if (is_x) { single_marker_freqs_and_hwe(unfiltered_sample_ctv2, loadbuf, sample_nonmale_include2, founder_nonmale_include2, founder_ctrl_nonmale_include2, founder_case_nonmale_include2, sample_nonmale_ct, &ll_ct, &lh_ct, &hh_ct, sample_f_nonmale_ct, &ll_ctf, &lh_ctf, &hh_ctf, hwe_needed, sample_f_ctl_nonmale_ct, &ll_hwe, &lh_hwe, &hh_hwe, hardy_needed, sample_f_case_nonmale_ct, &ll_case_hwe, &lh_case_hwe, &hh_case_hwe); hwe_ll_allfs[marker_uidx] = ll_ctf; hwe_lh_allfs[marker_uidx] = lh_ctf; hwe_hh_allfs[marker_uidx] = hh_ctf; uii = 2 * (ll_ctf + lh_ctf + hh_ctf); ujj = 2 * hh_ctf + lh_ctf; ukk = ll_ct + lh_ct + hh_ct; if (hwe_needed) { hwe_lls[marker_uidx] = ll_hwe; hwe_lhs[marker_uidx] = lh_hwe; hwe_hhs[marker_uidx] = hh_hwe; if (hardy_needed) { hwe_ll_cases[marker_uidx] = ll_case_hwe; hwe_lh_cases[marker_uidx] = lh_case_hwe; hwe_hh_cases[marker_uidx] = hh_case_hwe; } } } else if (!nonmissing_nonmale_y) { nonmissing_nonmale_y = nonmissing_present_diff(unfiltered_sample_ctv2, loadbuf, sample_include2, sample_male_include2); } // Quasi-bugfix, 13 Nov 2016: // There was a haploid_single_marker_freqs() function called by // calc_freqs_and_hwe(), which treated heterozygous haploids as // missing. This was inconsistent with PLINK 1.07, and also caused // --geno to be based on a different set of missing calls than // --mind. single_marker_freqs_and_hwe(unfiltered_sample_ctv2, loadbuf, sample_male_include2, founder_male_include2, nullptr, nullptr, sample_male_ct, &ll_ct, &lh_ct, &hh_ct, sample_f_male_ct, &ll_ctf, &lh_ctf, &hh_ctf, 0, 0, nullptr, nullptr, nullptr, 0, 0, nullptr, nullptr, nullptr); if ((is_x || sample_male_ct) && (sample_ct - cur_oblig_missing)) { if (is_x) { if (!cur_oblig_missing) { cur_genotyping_rate = ((int32_t)(ll_ct + lh_ct + hh_ct + ukk)) * sample_ct_recip; } else { cur_genotyping_rate = ((int32_t)(ll_ct + lh_ct + hh_ct + ukk)) / ((double)((int32_t)(sample_ct - cur_oblig_missing))); } } else { if (!cur_oblig_missing) { cur_genotyping_rate = ((int32_t)(ll_ct + lh_ct + hh_ct)) * male_ct_recip; } else { cur_genotyping_rate = ((int32_t)(ll_ct + lh_ct + hh_ct)) / ((double)((int32_t)(sample_male_ct - cur_oblig_missing))); } } } else { lh_ct = 0; cur_genotyping_rate = 0; nonmissing_rate_tot_max -= 1; } } else { single_marker_freqs_and_hwe(unfiltered_sample_ctv2, loadbuf, sample_include2, founder_include2, nullptr, nullptr, sample_ct, &ll_ct, &lh_ct, &hh_ct, sample_f_ct, &ll_ctf, &lh_ctf, &hh_ctf, 0, 0, nullptr, nullptr, nullptr, 0, 0, nullptr, nullptr, nullptr); if (!cur_oblig_missing) { cur_genotyping_rate = ((int32_t)(ll_ct + lh_ct + hh_ct)) * sample_ct_recip; } else { if (sample_ct - cur_oblig_missing) { cur_genotyping_rate = ((int32_t)(ll_ct + lh_ct + hh_ct)) / ((double)((int32_t)(sample_ct - cur_oblig_missing))); } else { cur_genotyping_rate = 0; nonmissing_rate_tot_max -= 1; } } } if (lh_ct) { if (!hhfile) { memcpy(outname_end, ".hh", 4); if (fopen_checked(outname, "w", &hhfile)) { goto calc_freqs_and_hwe_ret_OPEN_FAIL; } } if (is_x) { *hh_exists_ptr |= XMHH_EXISTS; } else if (is_y) { *hh_exists_ptr |= Y_FIX_NEEDED; } else { *hh_exists_ptr |= NXMHH_EXISTS; } if (is_x || is_y) { for (sample_uidx = 0; sample_uidx < unfiltered_sample_ctv2; sample_uidx++) { ulii = loadbuf[sample_uidx]; ulii = (ulii >> 1) & (~ulii) & sample_male_include2[sample_uidx]; while (ulii) { ukk = sample_uidx * BITCT2 + CTZLU(ulii) / 2; fputs(&(sample_ids[ukk * max_sample_id_len]), hhfile); putc_unlocked('\t', hhfile); fputs(&(marker_ids[marker_uidx * max_marker_id_len]), hhfile); putc_unlocked('\n', hhfile); ulii &= ulii - ONELU; } } } else { for (sample_uidx = 0; sample_uidx < unfiltered_sample_ctv2; sample_uidx++) { ulii = loadbuf[sample_uidx]; ulii = (ulii >> 1) & (~ulii) & sample_include2[sample_uidx]; while (ulii) { ukk = sample_uidx * BITCT2 + CTZLU(ulii) / 2; fputs(&(sample_ids[ukk * max_sample_id_len]), hhfile); putc_unlocked('\t', hhfile); fputs(&(marker_ids[marker_uidx * max_marker_id_len]), hhfile); putc_unlocked('\n', hhfile); ulii &= ulii - ONELU; } } } if (ferror(hhfile)) { goto calc_freqs_and_hwe_ret_WRITE_FAIL; } hethap_ct += lh_ct; } hwe_hapl_allfs[marker_uidx] = ll_ctf; hwe_haph_allfs[marker_uidx] = hh_ctf; uii += ll_ctf + hh_ctf; ujj += hh_ctf; if (ac_excl_bitfield) { if (ujj <= uii / 2) { ukk = ujj; } else { ukk = uii - ujj; } if ((ukk < min_ac) || (ukk > max_ac)) { set_bit(marker_uidx, ac_excl_bitfield); } } uii += 2 * maf_succ; ujj += maf_succ; if (!uii) { maf = 0.5; } else { maf = ((double)ujj) / ((double)uii); } set_allele_freqs[marker_uidx] = maf; } nonmissing_rate_tot += cur_genotyping_rate; if (geno_excl_bitfield && (cur_genotyping_rate < geno_thresh)) { SET_BIT(marker_uidx, geno_excl_bitfield); } } if (pct < 100) { if (pct > 10) { putc_unlocked('\b', stdout); } printf("\b\b%u%%", pct); fflush(stdout); } } fputs("\b\b\b\b", stdout); logprint(" done.\n"); if (hethap_ct) { *outname_end = '\0'; LOGERRPRINTFWW("Warning: %" PRIu64 " het. haploid genotype%s present (see %s.hh ); many commands treat these as missing.\n", hethap_ct, (hethap_ct == 1LLU)? "" : "s", outname); } if (nonmissing_nonmale_y) { logerrprint("Warning: Nonmissing nonmale Y chromosome genotype(s) present; many commands\ntreat these as missing.\n"); *hh_exists_ptr |= Y_FIX_NEEDED; } if (nonmissing_rate_tot <= 0.9999995 * ((double)((intptr_t)nonmissing_rate_tot_max))) { LOGPRINTF("Total genotyping rate %sis %g.\n", sample_exclude_ct? "in remaining samples " : "", nonmissing_rate_tot / ((double)((intptr_t)nonmissing_rate_tot_max))); } else { // oops, forgot this if (nonmissing_rate_tot == nonmissing_rate_tot_max) { LOGPRINTF("Total genotyping rate %sis exactly 1.\n", sample_exclude_ct? "in remaining samples " : ""); } else { LOGPRINTF("Total genotyping rate %sis in [0.9999995, 1).\n", sample_exclude_ct? "in remaining samples " : ""); } } while (0) { calc_freqs_and_hwe_ret_NOMEM: retval = RET_NOMEM; break; calc_freqs_and_hwe_ret_OPEN_FAIL: retval = RET_OPEN_FAIL; break; calc_freqs_and_hwe_ret_READ_FAIL: retval = RET_READ_FAIL; break; calc_freqs_and_hwe_ret_WRITE_FAIL: retval = RET_WRITE_FAIL; break; } bigstack_reset(bigstack_mark); fclose_cond(hhfile); return retval; } int32_t write_missingness_reports(FILE* bedfile, uintptr_t bed_offset, char* outname, char* outname_end, uint32_t output_gz, uint32_t plink_maxfid, uint32_t plink_maxiid, uint32_t plink_maxsnp, uintptr_t unfiltered_marker_ct, uintptr_t* marker_exclude, uintptr_t marker_ct, Chrom_info* chrom_info_ptr, Oblig_missing_info* om_ip, char* marker_ids, uintptr_t max_marker_id_len, uintptr_t unfiltered_sample_ct, uintptr_t sample_ct, uintptr_t* sample_exclude, uintptr_t* pheno_nm, uintptr_t* sex_male, uint32_t sample_male_ct, char* sample_ids, uintptr_t max_sample_id_len, uintptr_t cluster_ct, uint32_t* cluster_map, uint32_t* cluster_starts, char* cluster_ids, uintptr_t max_cluster_id_len, uint32_t hh_exists) { unsigned char* bigstack_mark = g_bigstack_base; uintptr_t unfiltered_sample_ct4 = (unfiltered_sample_ct + 3) / 4; uintptr_t unfiltered_sample_ctl2 = QUATERCT_TO_WORDCT(unfiltered_sample_ct); uintptr_t unfiltered_sample_ctv2 = round_up_pow2(unfiltered_sample_ctl2, 2); uintptr_t marker_ct_y = 0; uintptr_t* sample_male_include2 = nullptr; uint64_t* om_entry_ptr = nullptr; uintptr_t* cur_omidxs = nullptr; char* pzwritep = nullptr; uint32_t* sample_to_cluster = nullptr; uint32_t* missing_ct_by_cluster = nullptr; uint32_t* oblig_missing_ct_by_cluster = nullptr; uint32_t* cluster_sizes = nullptr; uint32_t* cluster_sizes_y = nullptr; uint32_t* om_cluster_sizes = nullptr; uint32_t* om_sample_lookup = nullptr; uint32_t* om_cluster_ref_cts = nullptr; uint64_t cur_om_entry = 0; int32_t y_code = chrom_info_ptr->xymt_codes[Y_OFFSET]; uint32_t y_present = (y_code != -2) && is_set(chrom_info_ptr->chrom_mask, y_code); uint32_t sample_uidx = 0; uint32_t sample_idx = 0; uint32_t oblig_ct = 0; uint32_t om_cluster_ct = 0; uint32_t om_cluster_ctl = 0; int32_t retval = 0; Pigz_state ps; uintptr_t* loadbuf; uintptr_t* sample_include2; uintptr_t* cur_nm; uintptr_t* lptr; uintptr_t* lptr2; uint32_t* missing_cts; uint32_t* cur_cluster_sizes; unsigned char* overflow_buf; char* cptr; char* cptr2; uintptr_t marker_ct_nony; uintptr_t marker_uidx; uintptr_t ulii; uint32_t slen; uint32_t sample_uidx_stop; uint32_t chrom_fo_idx; uint32_t chrom_idx; uint32_t chrom_end; uint32_t cur_tot; // uint32_t is_x; uint32_t is_y; // uint32_t is_haploid; uint32_t om_ycorr; uint32_t clidx; uint32_t uii; uint32_t ujj; uint32_t ukk; uint32_t umm; uint32_t unn; pzwrite_init_null(&ps); if (bigstack_alloc_uc(PIGZ_BLOCK_SIZE + MAXLINELEN, &overflow_buf) || bigstack_calloc_ui(unfiltered_sample_ct, &missing_cts) || bigstack_alloc_ul(unfiltered_sample_ctv2, &loadbuf) || bigstack_alloc_ul(unfiltered_sample_ctv2, &sample_include2) || bigstack_alloc_ul(unfiltered_sample_ctv2, &sample_male_include2)) { goto write_missingness_reports_ret_NOMEM; } loadbuf[unfiltered_sample_ctv2 - 2] = 0; loadbuf[unfiltered_sample_ctv2 - 1] = 0; init_quaterarr_from_inverted_bitarr(sample_exclude, unfiltered_sample_ct, sample_include2); memcpy(sample_male_include2, sample_include2, unfiltered_sample_ctv2 * sizeof(intptr_t)); apply_bitarr_mask_to_quaterarr_01(sex_male, unfiltered_sample_ct, sample_male_include2); if (y_present) { marker_ct_y = count_chrom_markers(chrom_info_ptr, marker_exclude, chrom_info_ptr->xymt_codes[Y_OFFSET]); } marker_ct_nony = marker_ct - marker_ct_y; if (fseeko(bedfile, bed_offset, SEEK_SET)) { goto write_missingness_reports_ret_READ_FAIL; } memcpy(outname_end, output_gz? ".lmiss.gz" : ".lmiss", output_gz? 10 : 7); if (flex_pzwrite_init(output_gz, outname, overflow_buf, 0, &ps)) { goto write_missingness_reports_ret_OPEN_FAIL; } pzwritep = (char*)overflow_buf; if (om_ip->entry_ct) { om_entry_ptr = om_ip->entries; om_cluster_ref_cts = om_ip->cluster_ref_cts; cur_om_entry = *om_entry_ptr; om_cluster_ct = om_ip->cluster_ct; om_cluster_ctl = BITCT_TO_WORDCT(om_cluster_ct); om_sample_lookup = om_ip->sample_lookup; // doubled because of Ychr if (bigstack_calloc_ui(om_cluster_ct * 2, &om_cluster_sizes)) { goto write_missingness_reports_ret_NOMEM; } for (sample_idx = 0; sample_idx < sample_ct; sample_uidx++, sample_idx++) { next_unset_unsafe_ck(sample_exclude, &sample_uidx); uii = om_sample_lookup[sample_uidx]; if (uii != 0xffffffffU) { om_cluster_sizes[uii] += 1; if (is_set(sex_male, sample_uidx)) { om_cluster_sizes[uii + om_cluster_ct] += 1; } } } sample_uidx = 0; sample_idx = 0; if (cluster_ct) { if (bigstack_alloc_ul(om_cluster_ctl, &cur_omidxs)) { goto write_missingness_reports_ret_NOMEM; } } } ujj = unfiltered_sample_ctl2 * BITCT2; if (!cluster_ct) { snprintf(g_textbuf, TEXTBUF_SIZE, " CHR %%%us N_MISS N_GENO F_MISS" EOLN_STR, plink_maxsnp); } else { if (bigstack_calloc_ui(unfiltered_sample_ct, &sample_to_cluster) || bigstack_alloc_ui(cluster_ct, &missing_ct_by_cluster) || bigstack_calloc_ui(cluster_ct, &oblig_missing_ct_by_cluster) || bigstack_calloc_ui(cluster_ct * 2, &cluster_sizes)) { goto write_missingness_reports_ret_NOMEM; } cluster_sizes_y = &(cluster_sizes[cluster_ct]); for (clidx = 0; clidx < cluster_ct; clidx++) { unn = cluster_starts[clidx + 1]; ukk = clidx + 1; for (uii = cluster_starts[clidx]; uii < unn; uii++) { umm = cluster_map[uii]; if (!IS_SET(sample_exclude, umm)) { sample_to_cluster[umm] = ukk; cluster_sizes[clidx] += 1; if (IS_SET(sex_male, umm)) { cluster_sizes_y[clidx] += 1; } } } } snprintf(g_textbuf, TEXTBUF_SIZE, " CHR %%%us CLST N_MISS N_CLST N_GENO F_MISS" EOLN_STR, plink_maxsnp); } pzwritep += sprintf(pzwritep, g_textbuf, "SNP"); for (chrom_fo_idx = 0; chrom_fo_idx < chrom_info_ptr->chrom_ct; chrom_fo_idx++) { chrom_idx = chrom_info_ptr->chrom_file_order[chrom_fo_idx]; chrom_end = chrom_info_ptr->chrom_fo_vidx_start[chrom_fo_idx + 1]; marker_uidx = next_unset(marker_exclude, chrom_info_ptr->chrom_fo_vidx_start[chrom_fo_idx], chrom_end); if (marker_uidx < chrom_end) { // is_x = (((int32_t)chrom_idx) == chrom_info_ptr->xymt_codes[X_OFFSET]); is_y = (((int32_t)chrom_idx) == chrom_info_ptr->xymt_codes[Y_OFFSET]); // is_haploid = is_set(chrom_info_ptr->haploid_mask, chrom_idx); if (!is_y) { cur_nm = sample_include2; cur_tot = sample_ct; cur_cluster_sizes = cluster_sizes; om_ycorr = 0; } else { cur_nm = sample_male_include2; cur_tot = sample_male_ct; cur_cluster_sizes = cluster_sizes_y; om_ycorr = om_cluster_ct; } cptr = width_force(4, g_textbuf, chrom_name_write(chrom_info_ptr, chrom_idx, g_textbuf)); *cptr++ = ' '; if (fseeko(bedfile, bed_offset + ((uint64_t)marker_uidx) * unfiltered_sample_ct4, SEEK_SET)) { goto write_missingness_reports_ret_READ_FAIL; } do { if (load_raw(unfiltered_sample_ct4, bedfile, loadbuf)) { goto write_missingness_reports_ret_READ_FAIL; } // if (is_haploid) { // haploid_fix(hh_exists, sample_include2, sample_male_include2, unfiltered_sample_ct, is_x, is_y, (unsigned char*)loadbuf); // } lptr = loadbuf; lptr2 = cur_nm; cptr2 = fw_strcpy(plink_maxsnp, &(marker_ids[marker_uidx * max_marker_id_len]), cptr); *cptr2++ = ' '; if (om_entry_ptr) { while ((cur_om_entry >> 32) < marker_uidx) { cur_om_entry = *(++om_entry_ptr); } if (cluster_ct) { fill_uint_zero(cluster_ct, oblig_missing_ct_by_cluster); } } if (!cluster_ct) { if (om_entry_ptr) { oblig_ct = 0; while ((cur_om_entry >> 32) == marker_uidx) { oblig_ct += om_cluster_sizes[(uint32_t)cur_om_entry]; cur_om_entry = *(++om_entry_ptr); } } ukk = 0; for (uii = 0; uii < ujj; uii += BITCT2) { ulii = *lptr++; ulii = ulii & ((~ulii) >> 1) & (*lptr2++); while (ulii) { missing_cts[uii + (CTZLU(ulii) / 2)] += 1; ukk++; ulii &= ulii - 1; } } pzwritep = memcpya(pzwritep, g_textbuf, cptr2 - g_textbuf); pzwritep = uint32toa_w8x(ukk - oblig_ct, ' ', pzwritep); pzwritep = uint32toa_w8x(cur_tot - oblig_ct, ' ', pzwritep); pzwritep = dtoa_g_wxp4(((double)((int32_t)(ukk - oblig_ct))) / ((double)((int32_t)(cur_tot - oblig_ct))), 8, pzwritep); append_binary_eoln(&pzwritep); if (flex_pzwrite(&ps, &pzwritep)) { goto write_missingness_reports_ret_WRITE_FAIL; } } else { fill_uint_zero(cluster_ct, missing_ct_by_cluster); if ((!om_entry_ptr) || ((cur_om_entry >> 32) != marker_uidx)) { for (uii = 0; uii < ujj; uii += BITCT2) { ulii = *lptr++; ulii = ulii & ((~ulii) >> 1) & (*lptr2++); while (ulii) { ukk = uii + (CTZLU(ulii) / 2); missing_cts[ukk] += 1; ukk = sample_to_cluster[ukk]; if (ukk) { missing_ct_by_cluster[ukk - 1] += 1; } ulii &= ulii - 1; } } } else { fill_ulong_zero(om_cluster_ctl, cur_omidxs); do { set_bit(((uint32_t)cur_om_entry) - om_ycorr, cur_omidxs); cur_om_entry = *(++om_entry_ptr); } while ((cur_om_entry >> 32) == marker_uidx); for (uii = 0; uii < ujj; uii += BITCT2) { ulii = *lptr++; ulii = ulii & ((~ulii) >> 1) & (*lptr2++); while (ulii) { ukk = uii + (CTZLU(ulii) / 2); missing_cts[ukk] += 1; umm = sample_to_cluster[ukk]; if (umm) { if (is_set(cur_omidxs, om_sample_lookup[ukk])) { oblig_missing_ct_by_cluster[umm - 1] += 1; } else { missing_ct_by_cluster[umm - 1] += 1; } } ulii &= ulii - 1; } } } for (clidx = 0; clidx < cluster_ct; clidx++) { pzwritep = memcpya(pzwritep, g_textbuf, cptr2 - g_textbuf); pzwritep = fw_strcpy(10, &(cluster_ids[clidx * max_cluster_id_len]), pzwritep); *pzwritep++ = ' '; uii = missing_ct_by_cluster[clidx]; pzwritep = uint32toa_w8x(uii, ' ', pzwritep); umm = cur_cluster_sizes[clidx]; pzwritep = uint32toa_w8x(umm, ' ', pzwritep); umm -= oblig_missing_ct_by_cluster[clidx]; pzwritep = uint32toa_w8x(umm, ' ', pzwritep); pzwritep = dtoa_g_wxp4(((double)((int32_t)uii)) / ((double)((int32_t)umm)), 8, pzwritep); append_binary_eoln(&pzwritep); if (flex_pzwrite(&ps, &pzwritep)) { goto write_missingness_reports_ret_WRITE_FAIL; } } } marker_uidx++; if (IS_SET(marker_exclude, marker_uidx)) { marker_uidx = next_unset_ul(marker_exclude, marker_uidx, chrom_end); if (marker_uidx < chrom_end) { if (fseeko(bedfile, bed_offset + ((uint64_t)marker_uidx) * unfiltered_sample_ct4, SEEK_SET)) { goto write_missingness_reports_ret_WRITE_FAIL; } } } } while (marker_uidx < chrom_end); } } if (flex_pzwrite_close_null(&ps, pzwritep)) { goto write_missingness_reports_ret_WRITE_FAIL; } outname_end[1] = 'i'; if (flex_pzwrite_init(output_gz, outname, overflow_buf, 0, &ps)) { goto write_missingness_reports_ret_OPEN_FAIL; } pzwritep = (char*)overflow_buf; snprintf(g_textbuf, TEXTBUF_SIZE, "%%%us %%%us MISS_PHENO N_MISS N_GENO F_MISS" EOLN_STR, plink_maxfid, plink_maxiid); pzwritep += sprintf(pzwritep, g_textbuf, "FID", "IID"); do { sample_uidx = next_unset_unsafe(sample_exclude, sample_uidx); sample_uidx_stop = next_set(sample_exclude, sample_uidx, unfiltered_sample_ct); sample_idx += sample_uidx_stop - sample_uidx; do { cptr = &(sample_ids[sample_uidx * max_sample_id_len]); cptr2 = (char*)memchr(cptr, '\t', max_sample_id_len); slen = (uintptr_t)(cptr2 - cptr); pzwritep = memseta(pzwritep, 32, plink_maxfid - slen); pzwritep = memcpyax(pzwritep, cptr, slen, ' '); pzwritep = fw_strcpy(plink_maxiid, &(cptr2[1]), pzwritep); pzwritep = memseta(pzwritep, 32, 10); *pzwritep++ = 'Y' - (is_set(pheno_nm, sample_uidx) * 11); *pzwritep++ = ' '; uii = missing_cts[sample_uidx]; ukk = is_set(sex_male, sample_uidx); ujj = marker_ct_nony + (ukk * marker_ct_y); if (om_sample_lookup) { umm = om_sample_lookup[sample_uidx]; if (umm != 0xffffffffU) { umm = om_cluster_ref_cts[umm] + ukk * om_cluster_ref_cts[umm + om_cluster_ct]; uii -= umm; ujj -= umm; } } pzwritep = uint32toa_w8x(uii, ' ', pzwritep); pzwritep = uint32toa_w8x(ujj, ' ', pzwritep); pzwritep = dtoa_g_wxp4(((double)((int32_t)uii)) / ((double)((int32_t)ujj)), 8, pzwritep); append_binary_eoln(&pzwritep); if (flex_pzwrite(&ps, &pzwritep)) { goto write_missingness_reports_ret_WRITE_FAIL; } } while (++sample_uidx < sample_uidx_stop); } while (sample_idx < sample_ct); if (flex_pzwrite_close_null(&ps, pzwritep)) { goto write_missingness_reports_ret_WRITE_FAIL; } *outname_end = '\0'; LOGPRINTFWW("--missing: Sample missing data report written to %s.imiss%s, and variant-based %smissing data report written to %s.lmiss%s.\n", outname, output_gz? ".gz" : "", cluster_ct? "cluster-stratified " : "", outname, output_gz? ".gz" : ""); while (0) { write_missingness_reports_ret_NOMEM: retval = RET_NOMEM; break; write_missingness_reports_ret_OPEN_FAIL: retval = RET_OPEN_FAIL; break; write_missingness_reports_ret_READ_FAIL: retval = RET_READ_FAIL; break; write_missingness_reports_ret_WRITE_FAIL: retval = RET_WRITE_FAIL; break; } bigstack_reset(bigstack_mark); flex_pzwrite_close_cond(&ps, pzwritep); return retval; } int32_t hardy_report_write_line(Pigz_state* ps_ptr, char** pzwritep_ptr, char* prefix_buf, uint32_t prefix_len, uint32_t reverse, uint32_t ll_ct, uint32_t lh_ct, uint32_t hh_ct, uint32_t hwe_midp, uint32_t is_mt, double pval, double output_min_p) { char* pzwritep = *pzwritep_ptr; char wbuf[48]; char* cptr; uint32_t denom; double drecip; double minor_freq; pzwritep = memcpya(pzwritep, prefix_buf, prefix_len); if (reverse) { cptr = uint32toa_x(hh_ct, '/', wbuf); cptr = uint32toa_x(lh_ct, '/', cptr); cptr = uint32toa(ll_ct, cptr); } else { cptr = uint32toa_x(ll_ct, '/', wbuf); cptr = uint32toa_x(lh_ct, '/', cptr); cptr = uint32toa(hh_ct, cptr); } pzwritep = fw_strcpyn(20, cptr - wbuf, wbuf, pzwritep); *pzwritep++ = ' '; denom = (ll_ct + lh_ct + hh_ct) * 2; if (denom && (!is_mt)) { drecip = 1.0 / ((double)denom); minor_freq = (2 * ll_ct + lh_ct) * drecip; pzwritep = dtoa_g_wxp4x((lh_ct * 2) * drecip, 8, ' ', pzwritep); pzwritep = dtoa_g_wxp4x(minor_freq * (2 * hh_ct + lh_ct) * drecip * 2, 8, ' ', pzwritep); pzwritep = dtoa_g_wxp4(MAXV(pval, output_min_p), 12, pzwritep); } else { pzwritep = memcpya(pzwritep, " nan nan ", 27); pzwritep = memcpyl3a(pzwritep, hwe_midp? "0.5" : " 1"); } append_binary_eoln(&pzwritep); if (flex_pzwrite(ps_ptr, &pzwritep)) { return 1; } *pzwritep_ptr = pzwritep; return 0; } int32_t hardy_report(char* outname, char* outname_end, double output_min_p, uintptr_t unfiltered_marker_ct, uintptr_t* marker_exclude, uintptr_t marker_exclude_ct, char* marker_ids, uintptr_t max_marker_id_len, uint32_t plink_maxsnp, char** marker_allele_ptrs, uintptr_t max_marker_allele_len, uintptr_t* marker_reverse, int32_t* hwe_lls, int32_t* hwe_lhs, int32_t* hwe_hhs, uint32_t hwe_modifier, uint32_t nonfounders, int32_t* hwe_ll_cases, int32_t* hwe_lh_cases, int32_t* hwe_hh_cases, int32_t* hwe_ll_allfs, int32_t* hwe_lh_allfs, int32_t* hwe_hh_allfs, uint32_t pheno_nm_ct, uintptr_t* pheno_c, Chrom_info* chrom_info_ptr) { unsigned char* bigstack_mark = g_bigstack_base; char* pzwritep = nullptr; uintptr_t marker_ct = unfiltered_marker_ct - marker_exclude_ct; uintptr_t marker_uidx = 0; uintptr_t marker_idx = 0; uint32_t hwe_midp = hwe_modifier & HWE_MIDP; uint32_t output_gz = (hwe_modifier / HWE_GZ) & 1; int32_t retval = 0; uint32_t pct = 0; Pigz_state ps; uint32_t prefix_len; uint32_t loop_end; uint32_t uii; uint32_t report_type; uint32_t is_x; uint32_t is_y; uint32_t is_mt; uint32_t is_haploid; uint32_t chrom_fo_idx; uint32_t chrom_end; uint32_t reverse; double* p_values; unsigned char* overflow_buf; char* writebuf; char* cptr0; char* cptr; char* cptr3; char* cptr4; char* cptr5; pzwrite_init_null(&ps); if (pheno_nm_ct) { report_type = pheno_c? 0 : 1; } else { report_type = 2; } uii = report_type? 1 : 3; if (bigstack_alloc_uc(PIGZ_BLOCK_SIZE + 2 * max_marker_allele_len + MAXLINELEN, &overflow_buf) || bigstack_alloc_d(uii * marker_ct, &p_values) || bigstack_alloc_c(2 * max_marker_allele_len + MAXLINELEN, &writebuf)) { goto hardy_report_ret_NOMEM; } // todo: multithread? if (report_type) { for (; marker_idx < marker_ct; marker_uidx++, marker_idx++) { next_unset_ul_unsafe_ck(marker_exclude, &marker_uidx); p_values[marker_idx] = SNPHWE2(hwe_lh_allfs[marker_uidx], hwe_ll_allfs[marker_uidx], hwe_hh_allfs[marker_uidx], hwe_midp); } } else { for (; marker_idx < marker_ct; marker_uidx++, marker_idx++) { next_unset_ul_unsafe_ck(marker_exclude, &marker_uidx); p_values[marker_idx * 3] = SNPHWE2(hwe_lh_allfs[marker_uidx], hwe_ll_allfs[marker_uidx], hwe_hh_allfs[marker_uidx], hwe_midp); p_values[marker_idx * 3 + 1] = SNPHWE2(hwe_lh_cases[marker_uidx], hwe_ll_cases[marker_uidx], hwe_hh_cases[marker_uidx], hwe_midp); p_values[marker_idx * 3 + 2] = SNPHWE2(hwe_lhs[marker_uidx], hwe_lls[marker_uidx], hwe_hhs[marker_uidx], hwe_midp); } } marker_uidx = 0; marker_idx = 0; memcpy(outname_end, output_gz? ".hwe.gz" : ".hwe", output_gz? 8 : 5); if (flex_pzwrite_init(output_gz, outname, overflow_buf, 0, &ps)) { goto hardy_report_ret_OPEN_FAIL; } pzwritep = (char*)overflow_buf; LOGPRINTFWW5("--hardy: Writing Hardy-Weinberg report (%s) to %s ... ", nonfounders? "all samples" : "founders only", outname); fputs("0%", stdout); fflush(stdout); sprintf(writebuf, " CHR %%%us TEST A1 A2 GENO O(HET) E(HET) P " EOLN_STR, plink_maxsnp); pzwritep += sprintf(pzwritep, writebuf, "SNP"); chrom_fo_idx = 0; refresh_chrom_info(chrom_info_ptr, marker_uidx, &chrom_end, &chrom_fo_idx, &is_x, &is_y, &is_mt, &is_haploid); cptr0 = width_force(4, writebuf, chrom_name_write(chrom_info_ptr, chrom_info_ptr->chrom_file_order[chrom_fo_idx], writebuf)); *cptr0++ = ' '; cptr = &(cptr0[10 + plink_maxsnp]); prefix_len = 10 + ((uintptr_t)(cptr - writebuf)); if (report_type) { if (report_type == 1) { memcpy(&(cptr0[plink_maxsnp]), " ALL(QT) ", 20); } else { memcpy(&(cptr0[plink_maxsnp]), " ALL(NP) ", 20); } for (; pct <= 100; pct++) { loop_end = (((uint64_t)pct) * marker_ct) / 100LLU; for (; marker_idx < loop_end; marker_uidx++, marker_idx++) { next_unset_ul_unsafe_ck(marker_exclude, &marker_uidx); if (marker_uidx >= chrom_end) { chrom_fo_idx++; refresh_chrom_info(chrom_info_ptr, marker_uidx, &chrom_end, &chrom_fo_idx, &is_x, &is_y, &is_mt, &is_haploid); cptr0 = width_force(4, writebuf, chrom_name_write(chrom_info_ptr, chrom_info_ptr->chrom_file_order[chrom_fo_idx], writebuf)); *cptr0++ = ' '; cptr = &(cptr0[10 + plink_maxsnp]); prefix_len = 10 + ((uintptr_t)(cptr - writebuf)); if (report_type == 1) { memcpy(&(cptr0[plink_maxsnp]), " ALL(QT) ", 20); } else { memcpy(&(cptr0[plink_maxsnp]), " ALL(NP) ", 20); } } fw_strcpy(plink_maxsnp, &(marker_ids[marker_uidx * max_marker_id_len]), cptr0); reverse = IS_SET(marker_reverse, marker_uidx); cptr3 = marker_allele_ptrs[2 * marker_uidx]; cptr4 = marker_allele_ptrs[2 * marker_uidx + 1]; cptr5 = fw_strcpy(4, cptr3, cptr); *cptr5 = ' '; cptr5 = fw_strcpy(4, cptr4, &(cptr5[1])); *cptr5 = ' '; prefix_len = 1 + (cptr5 - writebuf); if (hardy_report_write_line(&ps, &pzwritep, writebuf, prefix_len, reverse, hwe_ll_allfs[marker_uidx], hwe_lh_allfs[marker_uidx], hwe_hh_allfs[marker_uidx], hwe_midp, is_mt, p_values[marker_idx], output_min_p)) { goto hardy_report_ret_WRITE_FAIL; } } if (pct < 100) { if (pct > 10) { putc_unlocked('\b', stdout); } printf("\b\b%u%%", pct); fflush(stdout); } } } else { memset(&(cptr0[plink_maxsnp]), 32, 20); for (; pct <= 100; pct++) { loop_end = (((uint64_t)pct) * marker_ct) / 100LLU; for (; marker_idx < loop_end; marker_uidx++, marker_idx++) { next_unset_ul_unsafe_ck(marker_exclude, &marker_uidx); if (marker_uidx >= chrom_end) { chrom_fo_idx++; refresh_chrom_info(chrom_info_ptr, marker_uidx, &chrom_end, &chrom_fo_idx, &is_x, &is_y, &is_mt, &is_haploid); cptr0 = width_force(4, writebuf, chrom_name_write(chrom_info_ptr, chrom_info_ptr->chrom_file_order[chrom_fo_idx], writebuf)); *cptr0++ = ' '; memset(&(cptr0[plink_maxsnp]), 32, 20); cptr = &(cptr0[10 + plink_maxsnp]); prefix_len = 10 + ((uintptr_t)(cptr - writebuf)); } fw_strcpy(plink_maxsnp, &(marker_ids[marker_uidx * max_marker_id_len]), cptr0); memcpy(&(cptr0[4 + plink_maxsnp]), " ALL", 5); reverse = IS_SET(marker_reverse, marker_uidx); cptr3 = marker_allele_ptrs[2 * marker_uidx]; cptr4 = marker_allele_ptrs[2 * marker_uidx + 1]; cptr5 = fw_strcpy(4, cptr3, cptr); *cptr5 = ' '; cptr5 = fw_strcpy(4, cptr4, &(cptr5[1])); *cptr5 = ' '; prefix_len = 1 + (cptr5 - writebuf); if (hardy_report_write_line(&ps, &pzwritep, writebuf, prefix_len, reverse, hwe_ll_allfs[marker_uidx], hwe_lh_allfs[marker_uidx], hwe_hh_allfs[marker_uidx], hwe_midp, is_mt, p_values[3 * marker_idx], output_min_p)) { goto hardy_report_ret_WRITE_FAIL; } memcpy(&(cptr0[7 + plink_maxsnp]), "FF", 2); if (hardy_report_write_line(&ps, &pzwritep, writebuf, prefix_len, reverse, hwe_ll_cases[marker_uidx], hwe_lh_cases[marker_uidx], hwe_hh_cases[marker_uidx], hwe_midp, is_mt, p_values[3 * marker_idx + 1], output_min_p)) { goto hardy_report_ret_WRITE_FAIL; } memcpy(&(cptr0[4 + plink_maxsnp]), "UN", 2); if (hardy_report_write_line(&ps, &pzwritep, writebuf, prefix_len, reverse, hwe_lls[marker_uidx], hwe_lhs[marker_uidx], hwe_hhs[marker_uidx], hwe_midp, is_mt, p_values[3 * marker_idx + 2], output_min_p)) { goto hardy_report_ret_WRITE_FAIL; } } if (pct < 100) { if (pct > 10) { putc_unlocked('\b', stdout); } printf("\b\b%u%%", pct); fflush(stdout); } } } fputs("\b\b\b", stdout); logprint("done.\n"); while (0) { hardy_report_ret_NOMEM: retval = RET_NOMEM; break; hardy_report_ret_OPEN_FAIL: retval = RET_OPEN_FAIL; break; hardy_report_ret_WRITE_FAIL: retval = RET_WRITE_FAIL; break; } flex_pzwrite_close_cond(&ps, pzwritep); bigstack_reset(bigstack_mark); return retval; } uint32_t enforce_hwe_threshold(double hwe_thresh, uintptr_t unfiltered_marker_ct, uintptr_t* marker_exclude, uintptr_t* marker_exclude_ct_ptr, int32_t* hwe_lls, int32_t* hwe_lhs, int32_t* hwe_hhs, uint32_t hwe_modifier, uint32_t allow_no_variants, int32_t* hwe_ll_allfs, int32_t* hwe_lh_allfs, int32_t* hwe_hh_allfs, Chrom_info* chrom_info_ptr) { uint32_t marker_ct = unfiltered_marker_ct - *marker_exclude_ct_ptr; uint32_t marker_uidx = 0; uint32_t removed_ct = 0; uint32_t hwe_all = hwe_modifier & HWE_THRESH_ALL; uint32_t hwe_thresh_midp = hwe_modifier & HWE_THRESH_MIDP; uint32_t min_obs_nonx = 0xffffffffU; uint32_t min_obs_x = 0xffffffffU; uint32_t max_obs = 0; uint32_t chrom_fo_idx; uint32_t chrom_idx; uint32_t chrom_end; uint32_t cur_obs; uint32_t cur_min_obs; int32_t is_x; int32_t test_failed; if (chrom_info_ptr->haploid_mask[0] & 1) { logerrprint("Warning: --hwe has no effect since entire genome is haploid.\n"); return 0; } hwe_thresh *= 1 - SMALL_EPSILON; if (hwe_all) { hwe_lhs = hwe_lh_allfs; hwe_lls = hwe_ll_allfs; hwe_hhs = hwe_hh_allfs; } for (chrom_fo_idx = 0; chrom_fo_idx < chrom_info_ptr->chrom_ct; chrom_fo_idx++) { chrom_idx = chrom_info_ptr->chrom_file_order[chrom_fo_idx]; chrom_end = chrom_info_ptr->chrom_fo_vidx_start[chrom_fo_idx + 1]; marker_uidx = next_unset(marker_exclude, chrom_info_ptr->chrom_fo_vidx_start[chrom_fo_idx], chrom_end); if (marker_uidx < chrom_end) { is_x = (((int32_t)chrom_idx) == chrom_info_ptr->xymt_codes[X_OFFSET]); if ((((int32_t)chrom_idx) == chrom_info_ptr->xymt_codes[MT_OFFSET]) || (is_set(chrom_info_ptr->haploid_mask, chrom_idx) && (!is_x))) { continue; } // okay if min_obs_x is an underestimate cur_min_obs = min_obs_nonx; do { if (hwe_thresh_midp) { test_failed = SNPHWE_midp_t(hwe_lhs[marker_uidx], hwe_lls[marker_uidx], hwe_hhs[marker_uidx], hwe_thresh); } else { test_failed = SNPHWE_t(hwe_lhs[marker_uidx], hwe_lls[marker_uidx], hwe_hhs[marker_uidx], hwe_thresh); } if (test_failed) { SET_BIT(marker_uidx, marker_exclude); removed_ct++; } cur_obs = hwe_lhs[marker_uidx] + hwe_lls[marker_uidx] + hwe_hhs[marker_uidx]; if (cur_obs < cur_min_obs) { cur_min_obs = cur_obs; } if (cur_obs > max_obs) { max_obs = cur_obs; } marker_uidx = next_unset(marker_exclude, marker_uidx + 1, chrom_end); } while (marker_uidx < chrom_end); if (is_x) { min_obs_x = cur_min_obs; } else { min_obs_nonx = cur_min_obs; } } } if (((uint64_t)max_obs) * 9 > ((uint64_t)min_obs_nonx) * 10) { logerrprint("Warning: --hwe observation counts vary by more than 10%. Consider using\n--geno, and/or applying different p-value thresholds to distinct subsets of\nyour data.\n"); } else if (((uint64_t)max_obs) * 9 > ((uint64_t)min_obs_x) * 10) { logerrprint("Warning: --hwe observation counts vary by more than 10%, due to the X\nchromosome. You may want to use a more stringent (i.e. less extreme) --hwe\np-value threshold for X chromosome variants: male samples are ignored there, so\nthe same degree of HWE violation corresponds to a less-extreme p-value than it\ndoes elsewhere in the genome.\n"); } if ((marker_ct == removed_ct) && (!allow_no_variants)) { logerrprint("Error: All variants removed due to Hardy-Weinberg exact test (--hwe).\n"); return 1; } LOGPRINTF("--hwe: %u variant%s removed due to Hardy-Weinberg exact test.\n", removed_ct, (removed_ct == 1)? "" : "s"); *marker_exclude_ct_ptr += removed_ct; return 0; } uint32_t enforce_minor_allele_thresholds(double min_maf, double max_maf, uintptr_t unfiltered_marker_ct, uintptr_t* marker_exclude, uintptr_t* ac_excl_bitfield, uintptr_t* marker_exclude_ct_ptr, double* set_allele_freqs, uint32_t allow_no_variants) { uint32_t unfiltered_marker_ctl = BITCT_TO_WORDCT(unfiltered_marker_ct); uint32_t marker_ct = unfiltered_marker_ct - *marker_exclude_ct_ptr; uint32_t marker_uidx = 0; uint32_t removed_ct = 0; uint32_t markers_done = 0; uint32_t marker_uidx_stop; double dxx; if ((min_maf != 0.0) || (max_maf != 0.5)) { min_maf *= 1 - SMALL_EPSILON; max_maf *= 1 + SMALL_EPSILON; while (markers_done < marker_ct) { marker_uidx = next_unset_unsafe(marker_exclude, marker_uidx); marker_uidx_stop = next_set(marker_exclude, marker_uidx, unfiltered_marker_ct); markers_done += marker_uidx_stop - marker_uidx; do { dxx = get_maf(set_allele_freqs[marker_uidx]); if ((dxx < min_maf) || (dxx > max_maf)) { SET_BIT(marker_uidx, marker_exclude); } } while (++marker_uidx < marker_uidx_stop); } } if (ac_excl_bitfield) { bitvec_or(ac_excl_bitfield, unfiltered_marker_ctl, marker_exclude); } removed_ct = popcount_longs(marker_exclude, unfiltered_marker_ctl) - (*marker_exclude_ct_ptr); if ((marker_ct == removed_ct) && (!allow_no_variants)) { logerrprint("Error: All variants removed due to minor allele threshold(s)\n(--maf/--max-maf/--mac/--max-mac).\n"); return 1; } LOGPRINTFWW("%u variant%s removed due to minor allele threshold(s) (--maf/--max-maf/--mac/--max-mac).\n", removed_ct, (removed_ct == 1)? "" : "s"); *marker_exclude_ct_ptr += removed_ct; return 0; } void enforce_min_bp_space(int32_t min_bp_space, uint32_t unfiltered_marker_ct, uintptr_t* marker_exclude, uint32_t* marker_pos, uintptr_t* marker_exclude_ct_ptr, Chrom_info* chrom_info_ptr) { uint32_t marker_ct = unfiltered_marker_ct - (uint32_t)(*marker_exclude_ct_ptr); uint32_t chrom_ct = chrom_info_ptr->chrom_ct; uint32_t removed_ct = 0; uint32_t chrom_end = 0; uint32_t marker_uidx = next_unset(marker_exclude, 0, unfiltered_marker_ct); uint32_t chrom_fo_idx_p1 = 0; uint32_t marker_uidx_stop; int32_t last_pos; int32_t cur_pos; for (chrom_fo_idx_p1 = 1; chrom_fo_idx_p1 <= chrom_ct; chrom_fo_idx_p1++) { chrom_end = chrom_info_ptr->chrom_fo_vidx_start[chrom_fo_idx_p1]; if (marker_uidx >= chrom_end) { continue; } last_pos = -2147483647; do { marker_uidx_stop = next_set(marker_exclude, marker_uidx, chrom_end); do { cur_pos = marker_pos[marker_uidx]; if (cur_pos < last_pos + min_bp_space) { SET_BIT(marker_uidx, marker_exclude); removed_ct++; } else { last_pos = cur_pos; } } while (++marker_uidx < marker_uidx_stop); marker_uidx = next_unset(marker_exclude, marker_uidx, unfiltered_marker_ct); } while (marker_uidx < chrom_end); } LOGPRINTF("--bp-space: %u variant%s removed (%u remaining).\n", removed_ct, (removed_ct == 1)? "" : "s", marker_ct - removed_ct); *marker_exclude_ct_ptr += removed_ct; }