Debian Codegraph @

mysql-test/suite/ndb/t/ndb_import.pl

raw
1 use strict;
2 use Symbol;
3
4 my $vardir = $ENV{MYSQLTEST_VARDIR}
5 or die "need MYSQLTEST_VARDIR";
6
7 # fixed parameters
8
9 my $par_nullpct = 10; # pct nulls if nullable
10 my $par_sppct = 5; # a few pct special values like min,max
11 my $par_quotepct = 20; # pct quoted if fields_enclosed_by
12 my $par_valerrpct = 2 ; # pct value errors
13 my $par_randbias = 3; # prefer smaller random values
14
15 # type info
16
17 sub gen_int;
18 sub gen_bigint;
19 sub gen_float;
20 sub gen_char;
21 sub gen_decimal;
22 sub gen_bit;
23 sub gen_year;
24 sub gen_date;
25 sub gen_time;
26 sub gen_datetime;
27 sub gen_timestamp;
28 sub gen_blob;
29
30 my %typeinfo = (
31 tinyint => {
32 gen => \&gen_int,
33 min_signed => -2**7,
34 max_signed => 2**7-1,
35 max_unsigned => 2**8-1,
36 },
37 smallint => {
38 gen => \&gen_int,
39 min_signed => -2**15,
40 max_signed => 2**15-1,
41 max_unsigned => 2**16-1,
42 },
43 mediumint => {
44 gen => \&gen_int,
45 min_signed => -2**23,
46 max_signed => 2**23-1,
47 max_unsigned => 2**24-1,
48 },
49 int => {
50 gen => \&gen_int,
51 min_signed => -2**31,
52 max_signed => 2**31-1,
53 max_unsigned => 2**32-1,
54 },
55 bigint => {
56 gen => \&gen_bigint,
57 min_signed => -2**31,
58 max_signed => 2**31-1,
59 max_unsigned => 2**32-1,
60 },
61 double => {
62 gen => \&gen_float,
63 max_exp => 308,
64 },
65 float => {
66 gen => \&gen_float,
67 max_exp => 38
68 },
69 char => {
70 gen => \&gen_char,
71 },
72 varchar => {
73 gen => \&gen_char,
74 },
75 binary => {
76 binary => 1,
77 gen => \&gen_char,
78 },
79 varbinary => {
80 binary => 1,
81 gen => \&gen_char,
82 },
83 decimal => {
84 gen => \&gen_decimal,
85 },
86 bit => {
87 gen => \&gen_bit,
88 },
89 year => {
90 gen => \&gen_year,
91 },
92 date => {
93 gen => \&gen_date,
94 },
95 time => {
96 gen => \&gen_time,
97 },
98 datetime => {
99 gen => \&gen_datetime,
100 },
101 timestamp => {
102 gen => \&gen_timestamp,
103 },
104 text => {
105 gen => \&gen_blob,
106 },
107 blob => {
108 gen => \&gen_blob,
109 },
110 );
111
112 # init test
113
114 # CSV control strings
115 # note --exec uses sh so these are passed in single quotes
116 # note using --csvopt now to avoid quoting (windows)
117
118 sub get_csvfmt1 {
119 my $csvfmt = {
120 fields_terminated_by => '\t',
121 fields_enclosed_by => undef,
122 fields_escaped_by => '\\',
123 lines_terminated_by => '\n',
124 csvopt => 'n', # must match above
125 };
126 return $csvfmt;
127 }
128
129 sub get_csvfmt2 {
130 my $csvfmt = {
131 fields_terminated_by => ',',
132 fields_enclosed_by => '"',
133 fields_escaped_by => '\\',
134 lines_terminated_by => '\n',
135 csvopt => 'cqn', # must match above
136 };
137 return $csvfmt;
138 }
139
140 # translate used CSV control string to binary
141 # in perl '\\' and '\'' are 1 byte and other '\x' are 2 bytes
142 # basic double quoted chars work as expected
143
144 sub translate_csv_control {
145 my ($s) = @_;
146 return "\t" if $s eq '\t';
147 return "\n" if $s eq '\n';
148 return $s if $s eq ',';
149 return $s if $s eq '"';
150 return "\\" if $s eq '\\';
151 return "\\" if $s eq '\\\\';
152 die "translate_csv_control: unknown $s";
153 }
154
155 sub init_test {
156 my ($test) = @_;
157 my $f = $test->{csvfmt};
158 my $t = $test->{csvesc} = {};
159 my @name = qw(
160 fields_terminated_by
161 fields_enclosed_by
162 fields_escaped_by
163 lines_terminated_by
164 );
165 for my $name (@name) {
166 if (defined($f->{$name})) {
167 $t->{$name} = translate_csv_control($f->{$name});
168 } else {
169 $t->{$name} = undef;
170 }
171 }
172 # defaults
173 $test->{name} = "test->$test->{tag}";
174 $test->{statedir} = "$vardir/tmp";
175 $test->{csvdir} = "$vardir/tmp";
176 $test->{database} = "test";
177 if (defined($test->{verify})) {
178 $test->{verify} =~ /^(all|pk)$/
179 or die "$test->{name}: bad verify option '$test->{verify}'";
180 }
181 # does load data for verification need own csv file
182 $test->{csvver} = $test->{verify} && $test->{rejectsgen};
183 # resume requires bad rows
184 $test->{resumeopt} && !$test->{rejectsgen}
185 and die "$test->{name}: resume requires rejected rows";
186 }
187
188 sub get_tablename {
189 my ($test, $table, $opts) = @_;
190 my $tag = $test->{tag};
191 my $ver = $opts->{ver} ? "ver" : "";
192 return "$table->{name}$tag$ver";
193 }
194
195 sub get_csvfile {
196 my ($test, $table, $opts) = @_;
197 my $tag = $test->{tag};
198 my $ver = $test->{csvver} && $opts->{ver} ? "ver" : "";
199 return "$test->{csvdir}/$table->{name}$tag$ver.csv";
200 }
201
202 # create and drop
203
204 sub create_attr {
205 my ($attr) = @_;
206 $typeinfo{$attr->{type}}
207 or die "$attr->{name}: bad type: $attr->{type}";
208 my @txt = ();
209 push(@txt, $attr->{name});
210 push(@txt, $attr->{type});
211 push(@txt, "($attr->{len})")
212 if defined($attr->{len});
213 push(@txt, "($attr->{prec}, $attr->{scale})")
214 if defined($attr->{prec}) && defined($attr->{scale});
215 push(@txt, "($attr->{prec})")
216 if defined($attr->{prec}) && !defined($attr->{scale});
217 push(@txt, "unsigned")
218 if $attr->{unsigned};
219 push(@txt, "not null")
220 if $attr->{notnull};
221 return "@txt";
222 }
223
224 sub create_attrs {
225 my ($attrs) = @_;
226 my @txt = ();
227 my $attrno = 0;
228 for my $attr (@$attrs) {
229 $attr->{attrno} = $attrno++;
230 $attr->{notnull} = 1 if $attr->{pk};
231 }
232 for my $attr (@$attrs) {
233 my $s = $attr->{attrno} == 0 ? "\n" : ",\n";
234 push(@txt, $s . create_attr($attr));
235 $attrno++;
236 }
237 return "@txt";
238 }
239
240 sub create_table {
241 my ($test, $table, $opts) = @_;
242 my $tname = get_tablename($test, $table, $opts);
243 my @txt = ();
244 push(@txt, "create table $tname (");
245 my $attrs = $table->{attrs};
246 push(@txt, create_attrs($attrs));
247 my @pkattrs = ();
248 for my $attr (@$attrs) {
249 push(@pkattrs, $attr)
250 if $attr->{pk};
251 }
252 if (@pkattrs) {
253 my $pklist = join(", ", map($_->{name}, @pkattrs));
254 push(@txt, ",\nprimary key ($pklist)");
255 }
256 $table->{pkattrs} = [ @pkattrs ];
257 push(@txt, "\n) engine $opts->{engine};\n");
258 return "@txt";
259 }
260
261 sub create_tables {
262 my ($test, $opts) = @_;
263 my @txt = ();
264 my $tables = $test->{tables};
265 for my $table (@$tables) {
266 push(@txt, create_table($test, $table, $opts));
267 }
268 return "@txt";
269 };
270
271 sub drop_table {
272 my ($test, $table, $opts) = @_;
273 my $tname = get_tablename($test, $table, $opts);
274 my @txt = ();
275 push(@txt, "drop table");
276 push(@txt, "if exists")
277 if $opts->{if_exists};
278 push(@txt, "$tname;\n");
279 return "@txt";
280 }
281
282 sub drop_tables {
283 my ($test, $opts) = @_;
284 my @txt = ();
285 my $tables = $test->{tables};
286 for my $table (@$tables) {
287 push(@txt, drop_table($test, $table, $opts));
288 }
289 return "@txt";
290 };
291
292 # run test
293
294 sub run_import {
295 my ($test, $opts) = @_;
296 my $fter = $test->{csvfmt}{fields_terminated_by};
297 my $fenc = $test->{csvfmt}{fields_enclosed_by};
298 my @cmd = ();
299 push(@cmd, "\$NDB_IMPORT");
300 push(@cmd, "--state-dir=$test->{statedir}");
301 push(@cmd, "--keep-state");
302 push(@cmd, "--stats");
303 push(@cmd, "--input-type=csv");
304 push(@cmd, "--output-type=ndb");
305 push(@cmd, "--temperrors=100");
306 # using line terminator "\n"
307 push(@cmd, "--csvopt=n");
308 if (defined($test->{csvfmt}{csvopt})) {
309 push(@cmd, "--csvopt=$test->{csvfmt}{csvopt}");
310 } else {
311 push(@cmd, "--fields-terminated-by='$fter'");
312 if (defined($fenc)) {
313 push(@cmd, "--fields-optionally-enclosed-by='$fenc'");
314 }
315 }
316 if ($test->{rejectsopt}) {
317 push(@cmd, "--rejects=$test->{rejectsopt}");
318 }
319 # $opts tells if this is a resume
320 if ($opts->{resumeopt}) {
321 push(@cmd, "--resume");
322 }
323 push(@cmd, "--log-level=1");
324 push(@cmd, $test->{database});
325 my $tables = $test->{tables};
326 for my $table (@$tables) {
327 my $csvfile = get_csvfile($test, $table, $opts);
328 push(@cmd, $csvfile);
329 }
330 # runs mainly on unix/linux so stderr goes to stdout
331 my $log = '>>$NDB_TOOLS_OUTPUT 2>&1';
332 my @cmd1 = ("--exec", "echo", @cmd, $log);
333 my @cmd2 = ("--exec", @cmd, $log);
334 if (defined($opts->{error})) {
335 unshift(@cmd2, "--error $opts->{error}\n");
336 }
337 return "@cmd1\n@cmd2\n";
338 }
339
340 sub select_count {
341 my ($test, $table, $opts) = @_;
342 my $tname = get_tablename($test, $table, $opts);
343 my @txt = ();
344 push(@txt, "select count(*) from $tname;\n");
345 if ($test->{dumpdata}) {
346 my $file = "$vardir/tmp/$tname.dump";
347 push @txt, "--disable_query_log\n";
348 push(@txt, "select * from $tname order by 1\ninto outfile '$file';\n");
349 push @txt, "--enable_query_log\n";
350 }
351 return "@txt";
352 }
353
354 sub select_counts {
355 my ($test, $opts) = @_;
356 my @txt = ();
357 my $tables = $test->{tables};
358 for my $table (@$tables) {
359 push(@txt, select_count($test, $table, $opts));
360 }
361 return "@txt";
362 }
363
364 sub load_table {
365 my ($test, $table, $opts) = @_;
366 my $csvfile = get_csvfile($test, $table, $opts);
367 my $tname = get_tablename($test, $table, $opts);
368 my $fter = $test->{csvfmt}{fields_terminated_by};
369 my $fenc = $test->{csvfmt}{fields_enclosed_by};
370 my @txt = ();
371 push(@txt, "load data infile '$csvfile'\n");
372 push(@txt, "into table $tname\n");
373 # we generate 8-bit ascii so use some charset accepting it
374 push(@txt, "character set latin1\n");
375 push(@txt, "fields");
376 push(@txt, "terminated by '$fter'");
377 if (defined($fenc)) {
378 push(@txt, "optionally enclosed by '$fenc'");
379 }
380 push(@txt, "lines terminated by '\\n'");
381 push(@txt, ";\n");
382 return "@txt";
383 }
384
385 sub load_tables {
386 my ($test, $opts) = @_;
387 my @txt = ();
388 push @txt, "--disable_query_log\n";
389 my $tables = $test->{tables};
390 for my $table (@$tables) {
391 push(@txt, load_table($test, $table, $opts));
392 }
393 push @txt, "--enable_query_log\n";
394 return "@txt";
395 }
396
397 sub verify_table {
398 my ($test, $table, $opts) = @_;
399 my $tname1 = get_tablename($test, $table, { ver => 0 });
400 my $tname2 = get_tablename($test, $table, { ver => 1 });
401 my @txt = ();
402 push(@txt, "select count(*) from $tname1 x, $tname2 y\n");
403 push(@txt, "where");
404 my @cls = ();
405 my $attrs = $table->{attrs};
406 for my $attr (@$attrs) {
407 if ($test->{verify} eq 'pk' && !$attr->{pk}) {
408 next;
409 }
410 my $a1 = "x." . $attr->{name};
411 my $a2 = "y." . $attr->{name};
412 my $c;
413 if ($attr->{notnull}) {
414 $c = "$a1 = $a2";
415 } else {
416 $c = "($a1 = $a2 or ($a1 is null and $a2 is null))";
417 }
418 push(@cls, $c);
419 }
420 push(@txt, join(" and\n", @cls));
421 push(@txt, ";\n");
422 return "@txt";
423 }
424
425 sub verify_tables {
426 my ($test, $opts) = @_;
427 my @txt = ();
428 my $tables = $test->{tables};
429 for my $table (@$tables) {
430 push(@txt, verify_table($test, $table, $opts));
431 }
432 return "@txt";
433 }
434
435 # write test
436
437 sub make_test {
438 my ($test) = @_;
439 init_test($test);
440 my @txt = ();
441 push @txt, "--echo # test $test->{tag} - $test->{desc}\n";
442 push @txt, create_tables($test, { engine => "ndb" });
443 if (!$test->{resumeopt}) {
444 push @txt, run_import($test, {});
445 } else {
446 push @txt, run_import($test, { error => "1" });
447 for (my $i = 1; $i <= $test->{rejectsgen}; $i++) {
448 push @txt, run_import($test, { resumeopt => $i, error => "0,1" });
449 }
450 }
451 push @txt, select_counts($test, {});
452 if ($test->{verify}) {
453 push @txt, create_tables($test, { engine => "ndb", ver => 1 });
454 push @txt, load_tables($test, { ver => 1 });
455 push @txt, select_counts($test, { ver => 1 });
456 push @txt, verify_tables($test, {});
457 push @txt, drop_tables($test, { ver => 1 });
458 }
459 push @txt, drop_tables($test, {});
460 return "@txt";
461 }
462
463 sub write_tests {
464 my ($tests) = @_;
465 my $tag = $tests->{tag};
466 my $file = "$vardir/tmp/ndb_import$tag.inc";
467 my $fh = gensym();
468 open($fh, ">:raw", $file)
469 or die "$file: open for write failed: $!";
470 my $testlist = $tests->{testlist};
471 for my $test (@$testlist) {
472 print $fh make_test($test);
473 }
474 close($fh)
475 or die "$file: close after write failed: $!";
476 for my $test (@$testlist) {
477 # values in $opts propagate down *and* up
478 my $opts = {};
479 write_csvfiles($test, $opts);
480 }
481 }
482
483 # generate values
484
485 sub myrand {
486 my ($m) = @_;
487 return int(rand($m));
488 }
489
490 sub myrand2 {
491 my ($m, $k) = @_;
492 my $n = myrand($m);
493 while ($k > 0) {
494 $n = myrand($n + 1);
495 $k--;
496 }
497 return $n;
498 }
499
500 sub gen_int {
501 my ($test, $attr, $opts) = @_;
502 my $typeinfo = $typeinfo{$attr->{type}};
503 my $lo;
504 my $hi;
505 if ($attr->{unsigned}) {
506 $lo = 0;
507 $hi = $typeinfo->{max_unsigned};
508 } else {
509 $lo = $typeinfo->{min_signed};
510 $hi = $typeinfo->{max_signed};
511 }
512 my $val;
513 if ($attr->{pk}) {
514 $val = $opts->{rowid} % ($hi + 1);
515 } else {
516 my $r = myrand(100);
517 if ($r < $par_sppct) {
518 $val = 0 if $r % 3 == 0;
519 $val = $lo if $r % 3 == 1;
520 $val = $hi if $r % 3 == 2;
521 } else {
522 $val = $lo + myrand($hi - $lo + 1);
523 }
524 }
525 if ($attr->{type} eq 'smallint' &&
526 $opts->{rejectsflag} &&
527 myrand(100) < $par_valerrpct) {
528 if ($attr->{unsigned}) {
529 $val = $typeinfo->{max_unsigned} + 1;
530 } elsif (myrand(2) == 0) {
531 $val = $typeinfo->{max_signed} + 1;
532 } else {
533 $val = $typeinfo->{min_signed} - 1;
534 }
535 $opts->{rejectserrs} .= ":value";
536 }
537 return "$val";
538 }
539
540 sub gen_bigint {
541 my ($test, $attr, $opts) = @_;
542 my $val = gen_int($test, $attr, $opts);
543 return $val;
544 };
545
546 sub gen_float {
547 my ($test, $attr, $opts) = @_;
548 my $typeinfo = $typeinfo{$attr->{type}};
549 my $hiexp = $typeinfo->{max_exp};
550 my $p = myrand2($hiexp + 1, $par_randbias);
551 if (myrand(2) == 0) {
552 $p = (-1) * $p;
553 }
554 # floating rand
555 my $v = rand() * (10 ** $p);
556 if (myrand(2) == 0) {
557 $v = (-1) * $v;
558 }
559 my $val;
560 if (myrand(2) == 0) {
561 $val = sprintf("%f", $v);
562 } else {
563 $val = sprintf("%g", $v);
564 }
565 if ($attr->{type} eq 'float' &&
566 $opts->{rejectsflag} &&
567 myrand(100) < $par_valerrpct) {
568 $val = sprintf("1e%d", $hiexp + 10);
569 $opts->{rejectserrs} .= ":value";
570 }
571 return $val;
572 }
573
574 my $g_escape = {
575 0 => '0',
576 0x08 => 'b',
577 0x0a => 'n',
578 0x0d => 'r',
579 0x09 => 't',
580 0x1a => 'Z',
581 };
582
583 sub make_byte {
584 my ($test, $opts) = @_;
585 my $fter = $test->{csvesc}{fields_terminated_by};
586 my $fenc = $test->{csvesc}{fields_enclosed_by};
587 my $fesc = $test->{csvesc}{fields_escaped_by};
588 my $lter = $test->{csvesc}{lines_terminated_by};
589 my $binary = $opts->{binary};
590 my $mask = $opts->{mask};
591 my $val;
592 while (1) {
593 my $x;
594 # non-binary used only to make readable files, no charsets yet
595 if (!$binary) {
596 $x = 0x61 + myrand(26);
597 } else {
598 $x = myrand(256);
599 }
600 if (defined($mask)) {
601 $x &= $mask;
602 }
603 if ($x == 0) {
604 $val = $fesc.'0';
605 last;
606 }
607 if (0 && $x == 032) { # now using _O_BINARY on windows
608 $val = $fesc.'Z';
609 last;
610 }
611 if ($x == ord($fter)) {
612 if ($opts->{quote}) {
613 $val = $fter;
614 } else {
615 $val = $fesc.$fter;
616 }
617 last;
618 }
619 if (defined($fenc) && $x == ord($fenc)) {
620 if ($opts->{quote} && myrand(2) == 0) {
621 $val = $fenc.$fenc;
622 } else {
623 $val = $fesc.$fenc;
624 }
625 last;
626 }
627 if ($x == ord($fesc)) {
628 $val = $fesc.$fesc;
629 last;
630 }
631 if ($x == ord($lter)) {
632 if ($opts->{quote}) {
633 if (myrand(2) == 0) {
634 $val = $lter;
635 last;
636 }
637 }
638 if ($lter eq "\n") {
639 $val = "\\n";
640 last;
641 }
642 die "make_byte: cannot handle lter='$lter'";
643 }
644 if (myrand(5) == 0) {
645 my $v = $g_escape->{$x};
646 if (defined($v)) {
647 $val = $fesc.$v;
648 last;
649 }
650 }
651 $val = pack('C', $x);
652 last;
653 }
654 return $val;
655 }
656
657 sub make_string {
658 my ($test, $opts) = @_;
659 my $hilen = $opts->{hilen};
660 my $len = myrand(10) != 0 ? myrand2($hilen, $par_randbias) : $hilen;
661 if ($len == 0 && $opts->{minlen}) {
662 $len = $opts->{minlen};
663 }
664 my $val = "";
665 for (my $i = 0; $i < $len; $i++) {
666 $val .= make_byte($test, $opts);
667 }
668 return $val;
669 }
670
671 sub gen_char {
672 my ($test, $attr, $opts) = @_;
673 my $typeinfo = $typeinfo{$attr->{type}};
674 my $val;
675 if ($attr->{pk}) {
676 $val = sprintf("%d", $opts->{rowid});
677 } else {
678 $opts->{binary} = $typeinfo->{binary};
679 $opts->{hilen} = $attr->{len};
680 $opts->{minlen} = $attr->{minlen};
681 $val = make_string($test, $opts);
682 }
683 return $val;
684 }
685
686 sub gen_bit {
687 my ($test, $attr, $opts) = @_;
688 my $typeinfo = $typeinfo{$attr->{type}};
689 my $fter = $test->{csvfmt}{fields_terminated_by};
690 my $fesc = $test->{csvfmt}{fields_escaped_by};
691 my $l = $attr->{len};
692 my $n = int(($l + 7) / 8); # bytes rounded up
693 my $m = 8 - (8 * $n - $l); # bits in last byte
694 my $val = "";
695 for (my $i = 0; $i < $n; $i++) {
696 my $mask = 255;
697 if ($i + 1 == $n && $m != 0) {
698 $mask = (1 << $m) - 1;
699 }
700 $opts->{mask} = $mask;
701 my $v = make_byte($test, $opts);
702 $val .= $v;
703 }
704 return $val;
705 }
706
707 sub make_decimalpart {
708 my ($test, $opts) = @_;
709 my $n = $opts->{partlen};
710 my $m = myrand($n + 1);
711 my $val = "";
712 for (my $k = 0; $k < $m; $k++) {
713 my $d = myrand(10);
714 $val .= sprintf("%d", $d);
715 }
716 return $val;
717 }
718
719 sub gen_decimal {
720 my ($test, $attr, $opts) = @_;
721 my $prec = $attr->{prec};
722 my $scale = $attr->{scale};
723 $prec >= $scale or die "invalid prec=$prec scale=$scale";
724 my $val;
725 while (1) {
726 $opts->{partlen} = $prec - $scale;
727 $val = make_decimalpart($test, $opts);
728 if (myrand(2) == 0) {
729 $val .= ".";
730 if (myrand(2) == 0) {
731 $opts->{partlen} = $scale;
732 $val .= make_decimalpart($test, $opts);
733 }
734 }
735 last if $val =~ /\d/;
736 }
737 if ($attr->{unsigned}) {
738 if (myrand(3) == 0) {
739 $val = "+$val";
740 }
741 } else {
742 if (myrand(3) == 0) {
743 $val = "-$val";
744 }
745 }
746 return $val;
747 }
748
749 sub gen_year {
750 my ($test, $attr, $opts) = @_;
751 my $yy = 1901 + myrand(255);
752 my $val = sprintf("%04d", $yy);
753 return $val;
754 }
755
756 # Date::Manip could be used to generate better valid dates
757 sub make_date {
758 my ($test, $opts) = @_;
759 my $yystart = $opts->{yystart};
760 my $yyrange = $opts->{yyrange};
761 my $yy = $yystart + myrand($yyrange);
762 my $mm = 1 + myrand(12);
763 my $dd = 1 + myrand(28);
764 my $val = sprintf("%04d-%02d-%02d", $yy, $mm, $dd);
765 return $val;
766 }
767
768 sub gen_date {
769 my ($test, $attr, $opts) = @_;
770 $opts->{yystart} = 1950;
771 $opts->{yyrange} = 100;
772 my $val = make_date($test, $opts);
773 return $val;
774 }
775
776 sub make_frac {
777 my ($test, $opts) = @_;
778 my $prec = $opts->{prec};
779 my $n = myrand($prec + 1);
780 my $val = "";
781 for (my $i = 0; $i < $n; $i++) {
782 $val .= myrand(10);
783 }
784 $val = ".$val" if $n != 0 || myrand(5) == 0;
785 return $val;
786 }
787
788 sub make_time {
789 my ($test, $opts) = @_;
790 my $sep = $opts->{sep};
791 my $ts = $opts->{ts};
792 my $val;
793 my $hh = myrand(24);
794 if ($ts && ($hh == 2 || $hh == 3 || $hh == 4)) {
795 $hh = 1;
796 }
797 my $mm = myrand(60);
798 my $ss = myrand(60);
799 if ($sep) {
800 $hh = "0$hh" if $hh < 10 && myrand(2) == 0;
801 $mm = "0$mm" if $mm < 10 && myrand(2) == 0;
802 $ss = "0$ss" if $ss < 10 && myrand(2) == 0;
803 $val = "$hh:$mm:$ss";
804 } else {
805 $hh = "0$hh" if $hh < 10;
806 $mm = "0$mm" if $mm < 10;
807 $ss = "0$ss" if $ss < 10;
808 $val = "$hh$mm$ss";
809 }
810 $val .= make_frac($test, $opts);
811 return $val;
812 }
813
814 sub gen_time {
815 my ($test, $attr, $opts) = @_;
816 $opts->{sep} = myrand(2);
817 $opts->{prec} = $attr->{prec};
818 $opts->{ts} = 0;
819 my $val = make_time($test, $opts);
820 return $val;
821 }
822
823 sub gen_datetime {
824 my ($test, $attr, $opts) = @_;
825 $opts->{yystart} = 1950;
826 $opts->{yyrange} = 100;
827 my $d = make_date($test, $opts);
828 $opts->{sep} = 1;
829 $opts->{prec} = $attr->{prec};
830 $opts->{ts} = 0;
831 my $t = make_time($test, $opts);
832 my $p = " ";
833 return "$d$p$t"
834 }
835
836 sub gen_timestamp {
837 my ($test, $attr, $opts) = @_;
838 my $prec = $attr->{prec};
839 $opts->{yystart} = 1971;
840 $opts->{yyrange} = 64;
841 my $d = make_date($test, $opts);
842 $opts->{sep} = 1;
843 $opts->{prec} = $attr->{prec};
844 $opts->{ts} = 1;
845 my $t = make_time($test, $opts);
846 my $p = " ";
847 return "$d$p$t"
848 }
849
850 sub gen_blob {
851 my ($test, $attr, $opts) = @_;
852 my $typeinfo = $typeinfo{$attr->{type}};
853 $opts->{binary} = $typeinfo->{binary};
854 $opts->{hilen} = $attr->{len};
855 my $val = make_string($test, $opts);
856 return $val;
857 }
858
859 sub gen_csvfield {
860 my ($test, $table, $attr, $opts) = @_;
861 my $typeinfo = $typeinfo{$attr->{type}};
862 my $fenc = $test->{csvesc}{fields_enclosed_by};
863 my $fesc = $test->{csvesc}{fields_escaped_by};
864 my $val;
865 # if value will be enclosed by
866 $opts->{quote} = defined($fenc) &&
867 myrand(100) < $par_quotepct;
868 if (!$attr->{notnull} &&
869 myrand(100) < $par_nullpct) {
870 $val = $fesc.'N';
871 } else {
872 $val = $typeinfo->{gen}($test, $attr, $opts);
873 if ($opts->{rejectsflag} &&
874 myrand(100) < $par_nullpct) {
875 $val = $fesc.'N';
876 $opts->{rejectserrs} .= ":null";
877 }
878 }
879 if ($opts->{quote}) {
880 $val = "$fenc$val$fenc";
881 }
882 return $val;
883 }
884
885 sub get_pkvals {
886 my ($table, $line) = @_;
887 my $pkattrs = $table->{pkattrs};
888 my @pkvals = ();
889 for my $attr (@$pkattrs) {
890 my $n = $attr->{attrno};
891 push(@pkvals, $line->[$n]);
892 }
893 return [ @pkvals ];
894 }
895
896 sub set_pkvals {
897 my ($table, $line, $pkvals) = @_;
898 my $pkattrs = $table->{pkattrs};
899 my $k = 0;
900 for my $attr (@$pkattrs) {
901 my $n = $attr->{attrno};
902 $line->[$n] = $pkvals->[$k];
903 $k++;
904 }
905 }
906
907 sub gen_csvline {
908 my ($test, $table, $opts) = @_;
909 my $tname = get_tablename($test, $table, { ver => 0 });
910 my $attrs = $table->{attrs};
911 my $rowid = $opts->{rowid};
912 my $fter = $test->{csvesc}{fields_terminated_by};
913 my $fesc = $test->{csvesc}{fields_escaped_by};
914 my $lter = $test->{csvesc}{lines_terminated_by};
915 my @line = ();
916 # per line
917 $opts->{rejectserrs} = "";
918 for my $attr (@$attrs) {
919 my $val = gen_csvfield($test, $table, $attr, $opts);
920 push(@line, $val);
921 }
922 # if no errors from fields, create error on the row
923 if ($opts->{rejectsflag}) {
924 while ($opts->{rejectserrs} eq "") {
925 if (myrand(2) == 0 && $rowid > 0) {
926 # duplicate pk (one will be accepted, could be this one)
927 my $oldrowid = myrand($rowid);
928 my $oldpkvals = $opts->{pkvals}{$oldrowid};
929 if (defined($oldpkvals)) {
930 set_pkvals($table, \@line, $oldpkvals);
931 $opts->{rejectserrs} .= ":dubpk-$oldrowid";
932 last;
933 }
934 }
935 if (myrand(2) == 0) {
936 # too many fields
937 my $val = $fesc.'N';
938 push(@line, $val);
939 $opts->{rejectserrs} .= ":fields+1";
940 last;
941 }
942 if (myrand(2) == 0) {
943 # too few fields
944 pop(@line);
945 $opts->{rejectserrs} .= ":fields-1";
946 last;
947 }
948 }
949 }
950 # save generated rejects
951 ($opts->{rejectsflag} == ($opts->{rejectserrs} ne ""))
952 or die "rejects flag=$opts->{rejectsflag} errs=$opts->{rejectserrs}";
953 if ($opts->{rejectsflag}) {
954 $opts->{rejectslines}{$rowid} = [ @line, $opts->{rejectserrs} ];
955 }
956 # save pk values if line not rejected
957 if (!$opts->{rejectsflag})
958 {
959 my $pkvals = get_pkvals($table, \@line);
960 $opts->{pkvals}{$rowid} = $pkvals;
961 }
962 my $line = join($fter, @line).$lter;
963 return $line;
964 }
965
966 # write csv files
967
968 sub write_csvfile {
969 my ($test, $table, $opts) = @_;
970 my $file = get_csvfile($test, $table, { ver => 0 });
971 my $filever = get_csvfile($test, $table, { ver => 1 });
972 my $fh = gensym();
973 my $fhver = gensym();
974 open($fh, ">:raw", $file)
975 or die "$file: open for write failed: $!";
976 if ($test->{csvver}) {
977 open($fhver, ">:raw", $filever)
978 or die "$filever: open for write failed: $!";
979 }
980 my $tname = get_tablename($test, $table, { ver => 0 });
981 my $rows = $table->{rows};
982 # per file
983 $opts->{pkvals} = {};
984 $opts->{rejectscnt} = 0;
985 $opts->{rejectslines} = {};
986 for (my $n = 0; $n < $rows; $n++) {
987 $opts->{rowid} = $n;
988 $opts->{rejectsflag} = 0;
989 if ($test->{rejectsgen}) {
990 my $rowsleft = $rows - $n;
991 my $rejectsleft = $test->{rejectsgen} - $opts->{rejectscnt};
992 if ($rejectsleft != 0) {
993 if ($rejectsleft == $rowsleft ||
994 myrand(1 + $rowsleft/$rejectsleft) == 0) {
995 $opts->{rejectsflag} = 1;
996 $opts->{rejectserrs} = 0;
997 }
998 }
999 }
1000 my $line = gen_csvline($test, $table, $opts);
1001 if ($opts->{rejectsflag}) {
1002 $opts->{rejectserrs} or die "no rejectserrs";
1003 $opts->{rejectscnt}++;
1004 }
1005 print $fh "$line"
1006 or die "$file: rowid $n: write failed: $!";
1007 if ($test->{csvver}) {
1008 if (!$opts->{rejectsflag}) {
1009 print $fhver "$line"
1010 or die "$filever: rowid $n: write failed: $!";
1011 }
1012 }
1013 }
1014 if ($test->{rejectsgen}) {
1015 write_rejects($test, $table, $opts);
1016 }
1017 $test->{rejectsgen} == $opts->{rejectscnt}
1018 or die "$tname: $test->{rejectsgen} != $opts->{rejectscnt}";
1019 close($fh)
1020 or die "$file: close after write failed: $!";
1021 if ($test->{csvver}) {
1022 close($fhver)
1023 or die "$filever: close after write failed: $!";
1024 }
1025 }
1026
1027 # debug
1028 sub write_rejects {
1029 my ($test, $table, $opts) = @_;
1030 my $tname = get_tablename($test, $table, { ver => 0 });
1031 my $file = "$vardir/tmp/$tname.genrej";
1032 my $fh = gensym();
1033 open($fh, ">:raw", $file)
1034 or die "$file: open for write failed: $!";
1035 my $lines = $opts->{rejectslines};
1036 my @rowid = sort { $a <=> $b } keys %$lines;
1037 for my $rowid (@rowid) {
1038 my $line = $lines->{$rowid};
1039 my @out = join("\t", $rowid, @$line);
1040 print $fh @out, "\n";
1041 }
1042 close($fh)
1043 or die "$file: close after write failed: $!";
1044 }
1045
1046 sub write_csvfiles {
1047 my ($test, $opts) = @_;
1048 my $tables = $test->{tables};
1049 for my $table (@$tables) {
1050 write_csvfile($test, $table, $opts);
1051 }
1052 }
1053
1054 1;