| 1 |
use strict; |
| 2 |
use warnings; |
| 3 |
use autodie; |
| 4 |
|
| 5 |
use v5.16; |
| 6 |
|
| 7 |
use Getopt::Long; |
| 8 |
use JSON::XS; |
| 9 |
use Math::Int128 qw( uint128 ); |
| 10 |
use MaxMind::DB::Writer::Tree; |
| 11 |
use Memory::Stats; |
| 12 |
use Net::Works::Network; |
| 13 |
use Number::Format qw( format_bytes ); |
| 14 |
|
| 15 |
my $ipv6_offset = uint128('0xA000_0000_0000_0000_0000_0000_0000_0000'); |
| 16 |
|
| 17 |
sub main { |
| 18 |
my $inserts = 10_000; |
| 19 |
my $source; |
| 20 |
GetOptions( |
| 21 |
'inserts:i' => \$inserts, |
| 22 |
'source:s' => \$source, |
| 23 |
); |
| 24 |
|
| 25 |
my $stats = Memory::Stats->new(); |
| 26 |
$stats->start(); |
| 27 |
|
| 28 |
my $tree = MaxMind::DB::Writer::Tree->new( |
| 29 |
ip_version => 6, |
| 30 |
database_type => 'Test', |
| 31 |
description => { en => 'Test' }, |
| 32 |
languages => ['en'], |
| 33 |
map_key_type_callback => sub { 'utf8_string' }, |
| 34 |
record_size => 24, |
| 35 |
alias_ipv6_to_ipv4 => 1, |
| 36 |
); |
| 37 |
|
| 38 |
if ($source) { |
| 39 |
_insert_data_from_json( $tree, $inserts, $source ); |
| 40 |
} |
| 41 |
else { |
| 42 |
_insert_generated_data( $tree, $inserts ); |
| 43 |
} |
| 44 |
|
| 45 |
my $used; |
| 46 |
$stats->stop(); |
| 47 |
say 'Memory used before write: ', |
| 48 |
format_bytes( $used = $stats->get_memory_usage() ); |
| 49 |
|
| 50 |
$stats->start(); |
| 51 |
|
| 52 |
open my $fh, '>', '/dev/null'; |
| 53 |
$tree->write_tree($fh); |
| 54 |
close $fh; |
| 55 |
|
| 56 |
$stats->stop(); |
| 57 |
|
| 58 |
say 'Memory used after write: ', |
| 59 |
format_bytes( $used + $stats->get_memory_usage() ); |
| 60 |
} |
| 61 |
|
| 62 |
sub _insert_data_from_json { |
| 63 |
my $tree = shift; |
| 64 |
my $inserts = shift; |
| 65 |
my $source = shift; |
| 66 |
|
| 67 |
open my $fh, '<', $source; |
| 68 |
|
| 69 |
my $json = JSON::XS->new()->utf8(); |
| 70 |
|
| 71 |
while (<$fh>) { |
| 72 |
last if $. > $inserts; |
| 73 |
|
| 74 |
my ( $network, $record ) = @{ $json->decode($_) }; |
| 75 |
|
| 76 |
$tree->insert_network( |
| 77 |
Net::Works::Network->new_from_string( string => $network ), |
| 78 |
$record |
| 79 |
); |
| 80 |
} |
| 81 |
} |
| 82 |
|
| 83 |
sub _insert_generated_data { |
| 84 |
my $tree = shift; |
| 85 |
my $inserts = shift; |
| 86 |
|
| 87 |
for my $int ( 1 .. $inserts ) { |
| 88 |
my $ipv4 = Net::Works::Network->new_from_integer( |
| 89 |
integer => $int, |
| 90 |
prefix_length => 128, |
| 91 |
version => 6, |
| 92 |
); |
| 93 |
|
| 94 |
my %records; |
| 95 |
for my $i ( 1 .. 5 ) { |
| 96 |
$records{ipv4}{ 'thing' . $i } = { |
| 97 |
'ipv4_key1-' . $i => $int, |
| 98 |
'ipv4_key2-' . $i => { nested => "Foo $int-$i ipv4" }, |
| 99 |
}; |
| 100 |
|
| 101 |
$records{ipv6}{ 'thing' . $i } = { |
| 102 |
'ipv6_key1-' . $i => $int, |
| 103 |
'ipv6_key2-' . $i => { nested => "Foo $int-$i ipv6" }, |
| 104 |
}; |
| 105 |
} |
| 106 |
|
| 107 |
$tree->insert_network( $ipv4 => $records{ipv4} ); |
| 108 |
|
| 109 |
my $ipv6 = Net::Works::Network->new_from_integer( |
| 110 |
integer => $int + $ipv6_offset, |
| 111 |
mask_length => 128, |
| 112 |
); |
| 113 |
|
| 114 |
$tree->insert_network( $ipv6 => $records{ipv6} ); |
| 115 |
} |
| 116 |
} |
| 117 |
|
| 118 |
main(); |