| 1 |
/*- |
| 2 |
* Copyright 2013-2018,2022 Alexander Peslyak |
| 3 |
* All rights reserved. |
| 4 |
* |
| 5 |
* Redistribution and use in source and binary forms, with or without |
| 6 |
* modification, are permitted. |
| 7 |
* |
| 8 |
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND |
| 9 |
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| 10 |
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
| 11 |
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE |
| 12 |
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
| 13 |
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
| 14 |
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
| 15 |
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
| 16 |
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
| 17 |
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
| 18 |
* SUCH DAMAGE. |
| 19 |
*/ |
| 20 |
|
| 21 |
#ifdef __unix__ |
| 22 |
#include <sys/mman.h> |
| 23 |
#endif |
| 24 |
#ifdef __linux__ |
| 25 |
#include <linux/mman.h> /* for MAP_HUGE_2MB */ |
| 26 |
#endif |
| 27 |
|
| 28 |
#define HUGEPAGE_THRESHOLD (32 * 1024 * 1024) |
| 29 |
|
| 30 |
#ifdef __x86_64__ |
| 31 |
#define HUGEPAGE_SIZE (2 * 1024 * 1024) |
| 32 |
#else |
| 33 |
#undef HUGEPAGE_SIZE |
| 34 |
#endif |
| 35 |
|
| 36 |
static void *alloc_region(yescrypt_region_t *region, size_t size) |
| 37 |
{ |
| 38 |
size_t base_size = size; |
| 39 |
uint8_t *base, *aligned; |
| 40 |
#ifdef MAP_ANON |
| 41 |
unsigned int flags = |
| 42 |
#ifdef MAP_NOCORE |
| 43 |
MAP_NOCORE | |
| 44 |
#endif |
| 45 |
MAP_ANON | MAP_PRIVATE; |
| 46 |
#if defined(MAP_HUGETLB) && defined(MAP_HUGE_2MB) && defined(HUGEPAGE_SIZE) |
| 47 |
size_t new_size = size; |
| 48 |
const size_t hugepage_mask = (size_t)HUGEPAGE_SIZE - 1; |
| 49 |
if (size >= HUGEPAGE_THRESHOLD && size + hugepage_mask >= size) { |
| 50 |
flags |= MAP_HUGETLB | MAP_HUGE_2MB; |
| 51 |
/* |
| 52 |
* Linux's munmap() fails on MAP_HUGETLB mappings if size is not a multiple of |
| 53 |
* huge page size, so let's round up to huge page size here. |
| 54 |
*/ |
| 55 |
new_size = size + hugepage_mask; |
| 56 |
new_size &= ~hugepage_mask; |
| 57 |
} |
| 58 |
base = mmap(NULL, new_size, PROT_READ | PROT_WRITE, (int)flags, -1, 0); |
| 59 |
if (base != MAP_FAILED) { |
| 60 |
base_size = new_size; |
| 61 |
} else if (flags & MAP_HUGETLB) { |
| 62 |
flags &= ~(unsigned int)(MAP_HUGETLB | MAP_HUGE_2MB); |
| 63 |
base = mmap(NULL, size, PROT_READ | PROT_WRITE, (int)flags, -1, 0); |
| 64 |
} |
| 65 |
|
| 66 |
#else |
| 67 |
base = mmap(NULL, size, PROT_READ | PROT_WRITE, (int)flags, -1, 0); |
| 68 |
#endif |
| 69 |
if (base == MAP_FAILED) |
| 70 |
base = NULL; |
| 71 |
aligned = base; |
| 72 |
#else /* mmap not available */ |
| 73 |
base = aligned = NULL; |
| 74 |
if (size + 63 < size) { |
| 75 |
errno = ENOMEM; |
| 76 |
} else if ((base = malloc(size + 63)) != NULL) { |
| 77 |
aligned = base + 63; |
| 78 |
aligned -= (uintptr_t)aligned & 63; |
| 79 |
} |
| 80 |
#endif |
| 81 |
region->base = base; |
| 82 |
region->aligned = aligned; |
| 83 |
region->base_size = base ? base_size : 0; |
| 84 |
region->aligned_size = base ? size : 0; |
| 85 |
return aligned; |
| 86 |
} |
| 87 |
|
| 88 |
static inline void init_region(yescrypt_region_t *region) |
| 89 |
{ |
| 90 |
region->base = region->aligned = NULL; |
| 91 |
region->base_size = region->aligned_size = 0; |
| 92 |
} |
| 93 |
|
| 94 |
static int free_region(yescrypt_region_t *region) |
| 95 |
{ |
| 96 |
if (region->base) { |
| 97 |
#ifdef MAP_ANON |
| 98 |
if (munmap(region->base, region->base_size)) |
| 99 |
return -1; |
| 100 |
#else |
| 101 |
free(region->base); |
| 102 |
#endif |
| 103 |
} |
| 104 |
init_region(region); |
| 105 |
return 0; |
| 106 |
} |