| 1 |
/* |
| 2 |
Unix SMB/CIFS implementation. |
| 3 |
SMB torture tester |
| 4 |
Copyright (C) Andrew Tridgell 1997-2003 |
| 5 |
Copyright (C) Jelmer Vernooij 2006 |
| 6 |
|
| 7 |
This program is free software; you can redistribute it and/or modify |
| 8 |
it under the terms of the GNU General Public License as published by |
| 9 |
the Free Software Foundation; either version 3 of the License, or |
| 10 |
(at your option) any later version. |
| 11 |
|
| 12 |
This program is distributed in the hope that it will be useful, |
| 13 |
but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 14 |
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 15 |
GNU General Public License for more details. |
| 16 |
|
| 17 |
You should have received a copy of the GNU General Public License |
| 18 |
along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 19 |
*/ |
| 20 |
|
| 21 |
#include "includes.h" |
| 22 |
#include "lib/util/util_file.h" |
| 23 |
#include "libcli/raw/libcliraw.h" |
| 24 |
#include "libcli/raw/raw_proto.h" |
| 25 |
#include "system/time.h" |
| 26 |
#include "system/wait.h" |
| 27 |
#include "system/filesys.h" |
| 28 |
#include "../libcli/smb/smb_constants.h" |
| 29 |
#include "libcli/libcli.h" |
| 30 |
#include "lib/events/events.h" |
| 31 |
#include "libcli/resolve/resolve.h" |
| 32 |
#include "torture/smbtorture.h" |
| 33 |
#include "torture/util.h" |
| 34 |
#include "libcli/smb_composite/smb_composite.h" |
| 35 |
#include "libcli/composite/composite.h" |
| 36 |
#include "param/param.h" |
| 37 |
#include "torture/basic/proto.h" |
| 38 |
#include "lib/cmdline/cmdline.h" |
| 39 |
|
| 40 |
static bool wait_lock(struct smbcli_state *c, int fnum, uint32_t offset, uint32_t len) |
| 41 |
{ |
| 42 |
while (NT_STATUS_IS_ERR(smbcli_lock(c->tree, fnum, offset, len, -1, WRITE_LOCK))) { |
| 43 |
if (!check_error(__location__, c, ERRDOS, ERRlock, NT_STATUS_LOCK_NOT_GRANTED)) return false; |
| 44 |
} |
| 45 |
return true; |
| 46 |
} |
| 47 |
|
| 48 |
|
| 49 |
static bool rw_torture(struct torture_context *tctx, struct smbcli_state *c) |
| 50 |
{ |
| 51 |
const char *lockfname = "\\torture.lck"; |
| 52 |
char *fname; |
| 53 |
int fnum; |
| 54 |
int fnum2; |
| 55 |
pid_t pid2, pid = getpid(); |
| 56 |
int i, j; |
| 57 |
uint8_t buf[1024]; |
| 58 |
bool correct = true; |
| 59 |
|
| 60 |
fnum2 = smbcli_open(c->tree, lockfname, O_RDWR | O_CREAT | O_EXCL, |
| 61 |
DENY_NONE); |
| 62 |
if (fnum2 == -1) |
| 63 |
fnum2 = smbcli_open(c->tree, lockfname, O_RDWR, DENY_NONE); |
| 64 |
if (fnum2 == -1) { |
| 65 |
torture_comment(tctx, "open of %s failed (%s)\n", lockfname, smbcli_errstr(c->tree)); |
| 66 |
return false; |
| 67 |
} |
| 68 |
|
| 69 |
generate_random_buffer(buf, sizeof(buf)); |
| 70 |
|
| 71 |
for (i=0;i<torture_numops;i++) { |
| 72 |
unsigned int n = (unsigned int)random()%10; |
| 73 |
int ret; |
| 74 |
|
| 75 |
if (i % 10 == 0) { |
| 76 |
if (torture_setting_bool(tctx, "progress", true)) { |
| 77 |
torture_comment(tctx, "%d\r", i); |
| 78 |
fflush(stdout); |
| 79 |
} |
| 80 |
} |
| 81 |
ret = asprintf(&fname, "\\torture.%u", n); |
| 82 |
torture_assert(tctx, ret != -1, "asprintf failed"); |
| 83 |
|
| 84 |
if (!wait_lock(c, fnum2, n*sizeof(int), sizeof(int))) { |
| 85 |
return false; |
| 86 |
} |
| 87 |
|
| 88 |
fnum = smbcli_open(c->tree, fname, O_RDWR | O_CREAT | O_TRUNC, DENY_ALL); |
| 89 |
if (fnum == -1) { |
| 90 |
torture_comment(tctx, "open failed (%s)\n", smbcli_errstr(c->tree)); |
| 91 |
correct = false; |
| 92 |
break; |
| 93 |
} |
| 94 |
|
| 95 |
if (smbcli_write(c->tree, fnum, 0, &pid, 0, sizeof(pid)) != sizeof(pid)) { |
| 96 |
torture_comment(tctx, "write failed (%s)\n", smbcli_errstr(c->tree)); |
| 97 |
correct = false; |
| 98 |
} |
| 99 |
|
| 100 |
for (j=0;j<50;j++) { |
| 101 |
if (smbcli_write(c->tree, fnum, 0, buf, |
| 102 |
sizeof(pid)+(j*sizeof(buf)), |
| 103 |
sizeof(buf)) != sizeof(buf)) { |
| 104 |
torture_comment(tctx, "write failed (%s)\n", smbcli_errstr(c->tree)); |
| 105 |
correct = false; |
| 106 |
} |
| 107 |
} |
| 108 |
|
| 109 |
pid2 = 0; |
| 110 |
|
| 111 |
if (smbcli_read(c->tree, fnum, &pid2, 0, sizeof(pid)) != sizeof(pid)) { |
| 112 |
torture_comment(tctx, "read failed (%s)\n", smbcli_errstr(c->tree)); |
| 113 |
correct = false; |
| 114 |
} |
| 115 |
|
| 116 |
if (pid2 != pid) { |
| 117 |
torture_comment(tctx, "data corruption!\n"); |
| 118 |
correct = false; |
| 119 |
} |
| 120 |
|
| 121 |
if (NT_STATUS_IS_ERR(smbcli_close(c->tree, fnum))) { |
| 122 |
torture_comment(tctx, "close failed (%s)\n", smbcli_errstr(c->tree)); |
| 123 |
correct = false; |
| 124 |
} |
| 125 |
|
| 126 |
if (NT_STATUS_IS_ERR(smbcli_unlink(c->tree, fname))) { |
| 127 |
torture_comment(tctx, "unlink failed (%s)\n", smbcli_errstr(c->tree)); |
| 128 |
correct = false; |
| 129 |
} |
| 130 |
|
| 131 |
if (NT_STATUS_IS_ERR(smbcli_unlock(c->tree, fnum2, n*sizeof(int), sizeof(int)))) { |
| 132 |
torture_comment(tctx, "unlock failed (%s)\n", smbcli_errstr(c->tree)); |
| 133 |
correct = false; |
| 134 |
} |
| 135 |
free(fname); |
| 136 |
} |
| 137 |
|
| 138 |
smbcli_close(c->tree, fnum2); |
| 139 |
smbcli_unlink(c->tree, lockfname); |
| 140 |
|
| 141 |
torture_comment(tctx, "%d\n", i); |
| 142 |
|
| 143 |
return correct; |
| 144 |
} |
| 145 |
|
| 146 |
bool run_torture(struct torture_context *tctx, struct smbcli_state *cli, int dummy) |
| 147 |
{ |
| 148 |
return rw_torture(tctx, cli); |
| 149 |
} |
| 150 |
|
| 151 |
|
| 152 |
/* |
| 153 |
see how many RPC pipes we can open at once |
| 154 |
*/ |
| 155 |
bool run_pipe_number(struct torture_context *tctx, |
| 156 |
struct smbcli_state *cli1) |
| 157 |
{ |
| 158 |
const char *pipe_name = "\\WKSSVC"; |
| 159 |
int fnum; |
| 160 |
int num_pipes = 0; |
| 161 |
|
| 162 |
while(1) { |
| 163 |
fnum = smbcli_nt_create_full(cli1->tree, pipe_name, 0, SEC_FILE_READ_DATA, FILE_ATTRIBUTE_NORMAL, |
| 164 |
NTCREATEX_SHARE_ACCESS_READ|NTCREATEX_SHARE_ACCESS_WRITE, NTCREATEX_DISP_OPEN_IF, 0, 0); |
| 165 |
|
| 166 |
if (fnum == -1) { |
| 167 |
torture_comment(tctx, "Open of pipe %s failed with error (%s)\n", pipe_name, smbcli_errstr(cli1->tree)); |
| 168 |
break; |
| 169 |
} |
| 170 |
num_pipes++; |
| 171 |
if (torture_setting_bool(tctx, "progress", true)) { |
| 172 |
torture_comment(tctx, "%d\r", num_pipes); |
| 173 |
fflush(stdout); |
| 174 |
} |
| 175 |
} |
| 176 |
|
| 177 |
torture_comment(tctx, "pipe_number test - we can open %d %s pipes.\n", num_pipes, pipe_name ); |
| 178 |
return true; |
| 179 |
} |
| 180 |
|
| 181 |
|
| 182 |
|
| 183 |
|
| 184 |
/* |
| 185 |
open N connections to the server and just hold them open |
| 186 |
used for testing performance when there are N idle users |
| 187 |
already connected |
| 188 |
*/ |
| 189 |
bool torture_holdcon(struct torture_context *tctx) |
| 190 |
{ |
| 191 |
int i; |
| 192 |
struct smbcli_state **cli; |
| 193 |
int num_dead = 0; |
| 194 |
|
| 195 |
torture_comment(tctx, "Opening %d connections\n", torture_numops); |
| 196 |
|
| 197 |
cli = malloc_array_p(struct smbcli_state *, torture_numops); |
| 198 |
|
| 199 |
for (i=0;i<torture_numops;i++) { |
| 200 |
if (!torture_open_connection(&cli[i], tctx, i)) { |
| 201 |
return false; |
| 202 |
} |
| 203 |
if (torture_setting_bool(tctx, "progress", true)) { |
| 204 |
torture_comment(tctx, "opened %d connections\r", i+1); |
| 205 |
fflush(stdout); |
| 206 |
} |
| 207 |
} |
| 208 |
|
| 209 |
torture_comment(tctx, "\nStarting pings\n"); |
| 210 |
|
| 211 |
while (1) { |
| 212 |
for (i=0;i<torture_numops;i++) { |
| 213 |
NTSTATUS status; |
| 214 |
if (cli[i]) { |
| 215 |
status = smbcli_chkpath(cli[i]->tree, "\\"); |
| 216 |
if (!NT_STATUS_IS_OK(status)) { |
| 217 |
torture_comment(tctx, "Connection %d is dead\n", i); |
| 218 |
cli[i] = NULL; |
| 219 |
num_dead++; |
| 220 |
} |
| 221 |
usleep(100); |
| 222 |
} |
| 223 |
} |
| 224 |
|
| 225 |
if (num_dead == torture_numops) { |
| 226 |
torture_comment(tctx, "All connections dead - finishing\n"); |
| 227 |
break; |
| 228 |
} |
| 229 |
|
| 230 |
torture_comment(tctx, "."); |
| 231 |
fflush(stdout); |
| 232 |
} |
| 233 |
|
| 234 |
return true; |
| 235 |
} |
| 236 |
|
| 237 |
/* |
| 238 |
open a file N times on the server and just hold them open |
| 239 |
used for testing performance when there are N file handles |
| 240 |
open |
| 241 |
*/ |
| 242 |
bool torture_holdopen(struct torture_context *tctx, |
| 243 |
struct smbcli_state *cli) |
| 244 |
{ |
| 245 |
int i, fnum; |
| 246 |
const char *fname = "\\holdopen.dat"; |
| 247 |
NTSTATUS status; |
| 248 |
|
| 249 |
smbcli_unlink(cli->tree, fname); |
| 250 |
|
| 251 |
fnum = smbcli_open(cli->tree, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE); |
| 252 |
if (fnum == -1) { |
| 253 |
torture_comment(tctx, "open of %s failed (%s)\n", fname, smbcli_errstr(cli->tree)); |
| 254 |
return false; |
| 255 |
} |
| 256 |
|
| 257 |
smbcli_close(cli->tree, fnum); |
| 258 |
|
| 259 |
for (i=0;i<torture_numops;i++) { |
| 260 |
union smb_open op; |
| 261 |
|
| 262 |
op.generic.level = RAW_OPEN_NTCREATEX; |
| 263 |
op.ntcreatex.in.root_fid.fnum = 0; |
| 264 |
op.ntcreatex.in.flags = 0; |
| 265 |
op.ntcreatex.in.access_mask = SEC_FILE_WRITE_DATA; |
| 266 |
op.ntcreatex.in.create_options = 0; |
| 267 |
op.ntcreatex.in.file_attr = FILE_ATTRIBUTE_NORMAL; |
| 268 |
op.ntcreatex.in.share_access = NTCREATEX_SHARE_ACCESS_MASK; |
| 269 |
op.ntcreatex.in.alloc_size = 0; |
| 270 |
op.ntcreatex.in.open_disposition = NTCREATEX_DISP_OPEN; |
| 271 |
op.ntcreatex.in.impersonation = NTCREATEX_IMPERSONATION_ANONYMOUS; |
| 272 |
op.ntcreatex.in.security_flags = 0; |
| 273 |
op.ntcreatex.in.fname = fname; |
| 274 |
status = smb_raw_open(cli->tree, tctx, &op); |
| 275 |
if (!NT_STATUS_IS_OK(status)) { |
| 276 |
torture_warning(tctx, "open %d failed\n", i); |
| 277 |
continue; |
| 278 |
} |
| 279 |
|
| 280 |
if (torture_setting_bool(tctx, "progress", true)) { |
| 281 |
torture_comment(tctx, "opened %d file\r", i); |
| 282 |
fflush(stdout); |
| 283 |
} |
| 284 |
} |
| 285 |
|
| 286 |
torture_comment(tctx, "\nStarting pings\n"); |
| 287 |
|
| 288 |
while (1) { |
| 289 |
struct smb_echo ec; |
| 290 |
ZERO_STRUCT(ec); |
| 291 |
status = smb_raw_echo(cli->transport, &ec); |
| 292 |
torture_comment(tctx, "."); |
| 293 |
fflush(stdout); |
| 294 |
sleep(15); |
| 295 |
} |
| 296 |
} |
| 297 |
|
| 298 |
/* |
| 299 |
test how many open files this server supports on the one socket |
| 300 |
*/ |
| 301 |
bool torture_maxfid_test(struct torture_context *tctx, struct smbcli_state *cli) |
| 302 |
{ |
| 303 |
#define MAXFID_TEMPLATE "\\maxfid\\fid%d\\maxfid.%d.%d" |
| 304 |
char *fname; |
| 305 |
int fnums[0x11000], i; |
| 306 |
int retries=4, maxfid; |
| 307 |
bool correct = true; |
| 308 |
int ret; |
| 309 |
|
| 310 |
if (retries <= 0) { |
| 311 |
torture_comment(tctx, "failed to connect\n"); |
| 312 |
return false; |
| 313 |
} |
| 314 |
|
| 315 |
if (smbcli_deltree(cli->tree, "\\maxfid") == -1) { |
| 316 |
torture_comment(tctx, "Failed to deltree \\maxfid - %s\n", |
| 317 |
smbcli_errstr(cli->tree)); |
| 318 |
return false; |
| 319 |
} |
| 320 |
if (NT_STATUS_IS_ERR(smbcli_mkdir(cli->tree, "\\maxfid"))) { |
| 321 |
torture_comment(tctx, "Failed to mkdir \\maxfid, error=%s\n", |
| 322 |
smbcli_errstr(cli->tree)); |
| 323 |
return false; |
| 324 |
} |
| 325 |
|
| 326 |
torture_comment(tctx, "Testing maximum number of open files\n"); |
| 327 |
|
| 328 |
for (i=0; i<0x11000; i++) { |
| 329 |
if (i % 1000 == 0) { |
| 330 |
ret = asprintf(&fname, "\\maxfid\\fid%d", i/1000); |
| 331 |
torture_assert(tctx, ret != -1, "asprintf failed"); |
| 332 |
if (NT_STATUS_IS_ERR(smbcli_mkdir(cli->tree, fname))) { |
| 333 |
torture_comment(tctx, "Failed to mkdir %s, error=%s\n", |
| 334 |
fname, smbcli_errstr(cli->tree)); |
| 335 |
return false; |
| 336 |
} |
| 337 |
free(fname); |
| 338 |
} |
| 339 |
ret = asprintf(&fname, MAXFID_TEMPLATE, i/1000, i,(int)getpid()); |
| 340 |
torture_assert(tctx, ret != -1, "asprintf failed"); |
| 341 |
if ((fnums[i] = smbcli_open(cli->tree, fname, |
| 342 |
O_RDWR|O_CREAT|O_TRUNC, DENY_NONE)) == |
| 343 |
-1) { |
| 344 |
torture_comment(tctx, "open of %s failed (%s)\n", |
| 345 |
fname, smbcli_errstr(cli->tree)); |
| 346 |
torture_comment(tctx, "maximum fnum is %d\n", i); |
| 347 |
break; |
| 348 |
} |
| 349 |
free(fname); |
| 350 |
if (torture_setting_bool(tctx, "progress", true)) { |
| 351 |
torture_comment(tctx, "%6d\r", i); |
| 352 |
fflush(stdout); |
| 353 |
} |
| 354 |
} |
| 355 |
torture_comment(tctx, "%6d\n", i); |
| 356 |
|
| 357 |
maxfid = i; |
| 358 |
|
| 359 |
torture_comment(tctx, "cleaning up\n"); |
| 360 |
for (i=0;i<maxfid;i++) { |
| 361 |
ret = asprintf(&fname, MAXFID_TEMPLATE, i/1000, i,(int)getpid()); |
| 362 |
torture_assert(tctx, ret != -1, "asprintf failed"); |
| 363 |
if (NT_STATUS_IS_ERR(smbcli_close(cli->tree, fnums[i]))) { |
| 364 |
torture_comment(tctx, "Close of fnum %d failed - %s\n", fnums[i], smbcli_errstr(cli->tree)); |
| 365 |
} |
| 366 |
if (NT_STATUS_IS_ERR(smbcli_unlink(cli->tree, fname))) { |
| 367 |
torture_comment(tctx, "unlink of %s failed (%s)\n", |
| 368 |
fname, smbcli_errstr(cli->tree)); |
| 369 |
correct = false; |
| 370 |
} |
| 371 |
free(fname); |
| 372 |
|
| 373 |
if (torture_setting_bool(tctx, "progress", true)) { |
| 374 |
torture_comment(tctx, "%6d\r", i); |
| 375 |
fflush(stdout); |
| 376 |
} |
| 377 |
} |
| 378 |
torture_comment(tctx, "%6d\n", 0); |
| 379 |
|
| 380 |
if (smbcli_deltree(cli->tree, "\\maxfid") == -1) { |
| 381 |
torture_comment(tctx, "Failed to deltree \\maxfid - %s\n", |
| 382 |
smbcli_errstr(cli->tree)); |
| 383 |
return false; |
| 384 |
} |
| 385 |
|
| 386 |
torture_comment(tctx, "maxfid test finished\n"); |
| 387 |
|
| 388 |
return correct; |
| 389 |
#undef MAXFID_TEMPLATE |
| 390 |
} |
| 391 |
|
| 392 |
|
| 393 |
|
| 394 |
/* |
| 395 |
sees what IOCTLs are supported |
| 396 |
*/ |
| 397 |
bool torture_ioctl_test(struct torture_context *tctx, |
| 398 |
struct smbcli_state *cli) |
| 399 |
{ |
| 400 |
uint16_t device, function; |
| 401 |
int fnum; |
| 402 |
const char *fname = "\\ioctl.dat"; |
| 403 |
NTSTATUS status; |
| 404 |
union smb_ioctl parms; |
| 405 |
TALLOC_CTX *mem_ctx; |
| 406 |
|
| 407 |
mem_ctx = talloc_named_const(tctx, 0, "ioctl_test"); |
| 408 |
|
| 409 |
smbcli_unlink(cli->tree, fname); |
| 410 |
|
| 411 |
fnum = smbcli_open(cli->tree, fname, O_RDWR|O_CREAT|O_EXCL, DENY_NONE); |
| 412 |
if (fnum == -1) { |
| 413 |
torture_comment(tctx, "open of %s failed (%s)\n", fname, smbcli_errstr(cli->tree)); |
| 414 |
return false; |
| 415 |
} |
| 416 |
|
| 417 |
parms.ioctl.level = RAW_IOCTL_IOCTL; |
| 418 |
parms.ioctl.in.file.fnum = fnum; |
| 419 |
parms.ioctl.in.request = IOCTL_QUERY_JOB_INFO; |
| 420 |
status = smb_raw_ioctl(cli->tree, mem_ctx, &parms); |
| 421 |
torture_comment(tctx, "ioctl job info: %s\n", smbcli_errstr(cli->tree)); |
| 422 |
|
| 423 |
for (device=0;device<0x100;device++) { |
| 424 |
torture_comment(tctx, "Testing device=0x%x\n", device); |
| 425 |
for (function=0;function<0x100;function++) { |
| 426 |
parms.ioctl.in.request = (device << 16) | function; |
| 427 |
status = smb_raw_ioctl(cli->tree, mem_ctx, &parms); |
| 428 |
|
| 429 |
if (NT_STATUS_IS_OK(status)) { |
| 430 |
torture_comment(tctx, "ioctl device=0x%x function=0x%x OK : %d bytes\n", |
| 431 |
device, function, (int)parms.ioctl.out.blob.length); |
| 432 |
} |
| 433 |
} |
| 434 |
} |
| 435 |
|
| 436 |
return true; |
| 437 |
} |
| 438 |
|
| 439 |
static void benchrw_callback(struct smbcli_request *req); |
| 440 |
enum benchrw_stage { |
| 441 |
START, |
| 442 |
OPEN_CONNECTION, |
| 443 |
CLEANUP_TESTDIR, |
| 444 |
MK_TESTDIR, |
| 445 |
OPEN_FILE, |
| 446 |
INITIAL_WRITE, |
| 447 |
READ_WRITE_DATA, |
| 448 |
MAX_OPS_REACHED, |
| 449 |
ERROR, |
| 450 |
CLOSE_FILE, |
| 451 |
CLEANUP, |
| 452 |
FINISHED |
| 453 |
}; |
| 454 |
|
| 455 |
struct bench_params { |
| 456 |
struct unclist{ |
| 457 |
const char *host; |
| 458 |
const char *share; |
| 459 |
} **unc; |
| 460 |
const char *workgroup; |
| 461 |
int retry; |
| 462 |
unsigned int writeblocks; |
| 463 |
unsigned int blocksize; |
| 464 |
unsigned int writeratio; |
| 465 |
int num_parallel_requests; |
| 466 |
}; |
| 467 |
|
| 468 |
struct benchrw_state { |
| 469 |
struct torture_context *tctx; |
| 470 |
char *dname; |
| 471 |
char *fname; |
| 472 |
uint16_t fnum; |
| 473 |
int nr; |
| 474 |
struct smbcli_tree *cli; |
| 475 |
uint8_t *buffer; |
| 476 |
int writecnt; |
| 477 |
int readcnt; |
| 478 |
int completed; |
| 479 |
int num_parallel_requests; |
| 480 |
void *req_params; |
| 481 |
enum benchrw_stage mode; |
| 482 |
struct bench_params *lpcfg_params; |
| 483 |
}; |
| 484 |
|
| 485 |
/* |
| 486 |
init params using lpcfg_parm_xxx |
| 487 |
return number of unclist entries |
| 488 |
*/ |
| 489 |
static int init_benchrw_params(struct torture_context *tctx, |
| 490 |
struct bench_params *lpar) |
| 491 |
{ |
| 492 |
char **unc_list = NULL; |
| 493 |
int num_unc_names = 0, conn_index=0, empty_lines=0; |
| 494 |
const char *p; |
| 495 |
lpar->retry = torture_setting_int(tctx, "retry",3); |
| 496 |
lpar->blocksize = torture_setting_int(tctx, "blocksize",65535); |
| 497 |
lpar->writeblocks = torture_setting_int(tctx, "writeblocks",15); |
| 498 |
lpar->writeratio = torture_setting_int(tctx, "writeratio",5); |
| 499 |
lpar->num_parallel_requests = torture_setting_int( |
| 500 |
tctx, "parallel_requests", 5); |
| 501 |
lpar->workgroup = lpcfg_workgroup(tctx->lp_ctx); |
| 502 |
|
| 503 |
p = torture_setting_string(tctx, "unclist", NULL); |
| 504 |
if (p) { |
| 505 |
char *h, *s; |
| 506 |
unc_list = file_lines_load(p, &num_unc_names, 0, NULL); |
| 507 |
if (!unc_list || num_unc_names <= 0) { |
| 508 |
torture_comment(tctx, "Failed to load unc names list " |
| 509 |
"from '%s'\n", p); |
| 510 |
exit(1); |
| 511 |
} |
| 512 |
|
| 513 |
lpar->unc = talloc_array(tctx, struct unclist *, |
| 514 |
(num_unc_names-empty_lines)); |
| 515 |
for(conn_index = 0; conn_index < num_unc_names; conn_index++) { |
| 516 |
/* ignore empty lines */ |
| 517 |
if(strlen(unc_list[conn_index % num_unc_names])==0){ |
| 518 |
empty_lines++; |
| 519 |
continue; |
| 520 |
} |
| 521 |
if (!smbcli_parse_unc( |
| 522 |
unc_list[conn_index % num_unc_names], |
| 523 |
NULL, &h, &s)) { |
| 524 |
torture_comment( |
| 525 |
tctx, "Failed to parse UNC " |
| 526 |
"name %s\n", |
| 527 |
unc_list[conn_index % num_unc_names]); |
| 528 |
exit(1); |
| 529 |
} |
| 530 |
lpar->unc[conn_index-empty_lines] = |
| 531 |
talloc(tctx, struct unclist); |
| 532 |
lpar->unc[conn_index-empty_lines]->host = h; |
| 533 |
lpar->unc[conn_index-empty_lines]->share = s; |
| 534 |
} |
| 535 |
return num_unc_names-empty_lines; |
| 536 |
}else{ |
| 537 |
lpar->unc = talloc_array(tctx, struct unclist *, 1); |
| 538 |
lpar->unc[0] = talloc(tctx,struct unclist); |
| 539 |
lpar->unc[0]->host = torture_setting_string(tctx, "host", |
| 540 |
NULL); |
| 541 |
lpar->unc[0]->share = torture_setting_string(tctx, "share", |
| 542 |
NULL); |
| 543 |
return 1; |
| 544 |
} |
| 545 |
} |
| 546 |
|
| 547 |
/* |
| 548 |
Called when the reads & writes are finished. closes the file. |
| 549 |
*/ |
| 550 |
static NTSTATUS benchrw_close(struct torture_context *tctx, |
| 551 |
struct smbcli_request *req, |
| 552 |
struct benchrw_state *state) |
| 553 |
{ |
| 554 |
union smb_close close_parms; |
| 555 |
|
| 556 |
NT_STATUS_NOT_OK_RETURN(req->status); |
| 557 |
|
| 558 |
torture_comment(tctx, "Close file %d (%d)\n",state->nr,state->fnum); |
| 559 |
close_parms.close.level = RAW_CLOSE_CLOSE; |
| 560 |
close_parms.close.in.file.fnum = state->fnum ; |
| 561 |
close_parms.close.in.write_time = 0; |
| 562 |
state->mode=CLOSE_FILE; |
| 563 |
|
| 564 |
req = smb_raw_close_send(state->cli, &close_parms); |
| 565 |
NT_STATUS_HAVE_NO_MEMORY(req); |
| 566 |
/*register the callback function!*/ |
| 567 |
req->async.fn = benchrw_callback; |
| 568 |
req->async.private_data = state; |
| 569 |
|
| 570 |
return NT_STATUS_OK; |
| 571 |
} |
| 572 |
|
| 573 |
static NTSTATUS benchrw_readwrite(struct torture_context *tctx, |
| 574 |
struct benchrw_state *state); |
| 575 |
static void benchrw_callback(struct smbcli_request *req); |
| 576 |
|
| 577 |
static void benchrw_rw_callback(struct smbcli_request *req) |
| 578 |
{ |
| 579 |
struct benchrw_state *state = req->async.private_data; |
| 580 |
struct torture_context *tctx = state->tctx; |
| 581 |
|
| 582 |
if (!NT_STATUS_IS_OK(req->status)) { |
| 583 |
state->mode = ERROR; |
| 584 |
return; |
| 585 |
} |
| 586 |
|
| 587 |
state->completed++; |
| 588 |
state->num_parallel_requests--; |
| 589 |
|
| 590 |
if ((state->completed >= torture_numops) |
| 591 |
&& (state->num_parallel_requests == 0)) { |
| 592 |
benchrw_callback(req); |
| 593 |
talloc_free(req); |
| 594 |
return; |
| 595 |
} |
| 596 |
|
| 597 |
talloc_free(req); |
| 598 |
|
| 599 |
if (state->completed + state->num_parallel_requests |
| 600 |
< torture_numops) { |
| 601 |
benchrw_readwrite(tctx, state); |
| 602 |
} |
| 603 |
} |
| 604 |
|
| 605 |
/* |
| 606 |
Called when the initial write is completed is done. write or read a file. |
| 607 |
*/ |
| 608 |
static NTSTATUS benchrw_readwrite(struct torture_context *tctx, |
| 609 |
struct benchrw_state *state) |
| 610 |
{ |
| 611 |
struct smbcli_request *req; |
| 612 |
union smb_read rd; |
| 613 |
union smb_write wr; |
| 614 |
|
| 615 |
/* randomize between writes and reads*/ |
| 616 |
if (random() % state->lpcfg_params->writeratio == 0) { |
| 617 |
torture_comment(tctx, "Callback WRITE file:%d (%d/%d)\n", |
| 618 |
state->nr,state->completed,torture_numops); |
| 619 |
wr.generic.level = RAW_WRITE_WRITEX ; |
| 620 |
wr.writex.in.file.fnum = state->fnum ; |
| 621 |
wr.writex.in.offset = 0; |
| 622 |
wr.writex.in.wmode = 0 ; |
| 623 |
wr.writex.in.remaining = 0; |
| 624 |
wr.writex.in.count = state->lpcfg_params->blocksize; |
| 625 |
wr.writex.in.data = state->buffer; |
| 626 |
state->readcnt=0; |
| 627 |
req = smb_raw_write_send(state->cli,&wr); |
| 628 |
} |
| 629 |
else { |
| 630 |
torture_comment(tctx, |
| 631 |
"Callback READ file:%d (%d/%d) Offset:%d\n", |
| 632 |
state->nr,state->completed,torture_numops, |
| 633 |
(state->readcnt*state->lpcfg_params->blocksize)); |
| 634 |
rd.generic.level = RAW_READ_READX; |
| 635 |
rd.readx.in.file.fnum = state->fnum ; |
| 636 |
rd.readx.in.offset = state->readcnt*state->lpcfg_params->blocksize; |
| 637 |
rd.readx.in.mincnt = state->lpcfg_params->blocksize; |
| 638 |
rd.readx.in.maxcnt = rd.readx.in.mincnt; |
| 639 |
rd.readx.in.remaining = 0 ; |
| 640 |
rd.readx.out.data = state->buffer; |
| 641 |
rd.readx.in.read_for_execute = false; |
| 642 |
if(state->readcnt < state->lpcfg_params->writeblocks){ |
| 643 |
state->readcnt++; |
| 644 |
}else{ |
| 645 |
/*start reading from beginning of file*/ |
| 646 |
state->readcnt=0; |
| 647 |
} |
| 648 |
req = smb_raw_read_send(state->cli,&rd); |
| 649 |
} |
| 650 |
state->num_parallel_requests += 1; |
| 651 |
NT_STATUS_HAVE_NO_MEMORY(req); |
| 652 |
/*register the callback function!*/ |
| 653 |
req->async.fn = benchrw_rw_callback; |
| 654 |
req->async.private_data = state; |
| 655 |
|
| 656 |
return NT_STATUS_OK; |
| 657 |
} |
| 658 |
|
| 659 |
/* |
| 660 |
Called when the open is done. writes to the file. |
| 661 |
*/ |
| 662 |
static NTSTATUS benchrw_open(struct torture_context *tctx, |
| 663 |
struct smbcli_request *req, |
| 664 |
struct benchrw_state *state) |
| 665 |
{ |
| 666 |
union smb_write wr; |
| 667 |
if(state->mode == OPEN_FILE){ |
| 668 |
NTSTATUS status; |
| 669 |
status = smb_raw_open_recv(req,tctx,( |
| 670 |
union smb_open*)state->req_params); |
| 671 |
NT_STATUS_NOT_OK_RETURN(status); |
| 672 |
|
| 673 |
state->fnum = ((union smb_open*)state->req_params) |
| 674 |
->openx.out.file.fnum; |
| 675 |
torture_comment(tctx, "File opened (%d)\n",state->fnum); |
| 676 |
state->mode=INITIAL_WRITE; |
| 677 |
} |
| 678 |
|
| 679 |
torture_comment(tctx, "Write initial test file:%d (%d/%d)\n",state->nr, |
| 680 |
(state->writecnt+1)*state->lpcfg_params->blocksize, |
| 681 |
(state->lpcfg_params->writeblocks*state->lpcfg_params->blocksize)); |
| 682 |
wr.generic.level = RAW_WRITE_WRITEX ; |
| 683 |
wr.writex.in.file.fnum = state->fnum ; |
| 684 |
wr.writex.in.offset = state->writecnt * |
| 685 |
state->lpcfg_params->blocksize; |
| 686 |
wr.writex.in.wmode = 0 ; |
| 687 |
wr.writex.in.remaining = (state->lpcfg_params->writeblocks * |
| 688 |
state->lpcfg_params->blocksize)- |
| 689 |
((state->writecnt+1)*state-> |
| 690 |
lpcfg_params->blocksize); |
| 691 |
wr.writex.in.count = state->lpcfg_params->blocksize; |
| 692 |
wr.writex.in.data = state->buffer; |
| 693 |
state->writecnt++; |
| 694 |
if(state->writecnt == state->lpcfg_params->writeblocks){ |
| 695 |
state->mode=READ_WRITE_DATA; |
| 696 |
} |
| 697 |
req = smb_raw_write_send(state->cli,&wr); |
| 698 |
NT_STATUS_HAVE_NO_MEMORY(req); |
| 699 |
|
| 700 |
/*register the callback function!*/ |
| 701 |
req->async.fn = benchrw_callback; |
| 702 |
req->async.private_data = state; |
| 703 |
return NT_STATUS_OK; |
| 704 |
} |
| 705 |
|
| 706 |
/* |
| 707 |
Called when the mkdir is done. Opens a file. |
| 708 |
*/ |
| 709 |
static NTSTATUS benchrw_mkdir(struct torture_context *tctx, |
| 710 |
struct smbcli_request *req, |
| 711 |
struct benchrw_state *state) |
| 712 |
{ |
| 713 |
union smb_open *open_parms; |
| 714 |
uint8_t *writedata; |
| 715 |
|
| 716 |
NT_STATUS_NOT_OK_RETURN(req->status); |
| 717 |
|
| 718 |
/* open/create the files */ |
| 719 |
torture_comment(tctx, "Open File %d/%d\n",state->nr+1, |
| 720 |
torture_setting_int(tctx, "nprocs", 4)); |
| 721 |
open_parms=talloc_zero(tctx, union smb_open); |
| 722 |
NT_STATUS_HAVE_NO_MEMORY(open_parms); |
| 723 |
open_parms->openx.level = RAW_OPEN_OPENX; |
| 724 |
open_parms->openx.in.flags = 0; |
| 725 |
open_parms->openx.in.open_mode = OPENX_MODE_ACCESS_RDWR; |
| 726 |
open_parms->openx.in.search_attrs = |
| 727 |
FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_HIDDEN; |
| 728 |
open_parms->openx.in.file_attrs = 0; |
| 729 |
open_parms->openx.in.write_time = 0; |
| 730 |
open_parms->openx.in.open_func = OPENX_OPEN_FUNC_CREATE; |
| 731 |
open_parms->openx.in.size = 0; |
| 732 |
open_parms->openx.in.timeout = 0; |
| 733 |
open_parms->openx.in.fname = state->fname; |
| 734 |
|
| 735 |
writedata = talloc_size(tctx,state->lpcfg_params->blocksize); |
| 736 |
NT_STATUS_HAVE_NO_MEMORY(writedata); |
| 737 |
generate_random_buffer(writedata,state->lpcfg_params->blocksize); |
| 738 |
state->buffer=writedata; |
| 739 |
state->writecnt=1; |
| 740 |
state->readcnt=0; |
| 741 |
state->req_params=open_parms; |
| 742 |
state->mode=OPEN_FILE; |
| 743 |
|
| 744 |
req = smb_raw_open_send(state->cli,open_parms); |
| 745 |
NT_STATUS_HAVE_NO_MEMORY(req); |
| 746 |
|
| 747 |
/*register the callback function!*/ |
| 748 |
req->async.fn = benchrw_callback; |
| 749 |
req->async.private_data = state; |
| 750 |
|
| 751 |
return NT_STATUS_OK; |
| 752 |
} |
| 753 |
|
| 754 |
/* |
| 755 |
handler for completion of a sub-request of the bench-rw test |
| 756 |
*/ |
| 757 |
static void benchrw_callback(struct smbcli_request *req) |
| 758 |
{ |
| 759 |
struct benchrw_state *state = req->async.private_data; |
| 760 |
struct torture_context *tctx = state->tctx; |
| 761 |
|
| 762 |
/*don't send new requests when torture_numops is reached*/ |
| 763 |
if ((state->mode == READ_WRITE_DATA) |
| 764 |
&& (state->completed >= torture_numops)) { |
| 765 |
state->mode=MAX_OPS_REACHED; |
| 766 |
} |
| 767 |
|
| 768 |
switch (state->mode) { |
| 769 |
|
| 770 |
case MK_TESTDIR: |
| 771 |
if (!NT_STATUS_IS_OK(benchrw_mkdir(tctx, req,state))) { |
| 772 |
torture_comment(tctx, "Failed to create the test " |
| 773 |
"directory - %s\n", |
| 774 |
nt_errstr(req->status)); |
| 775 |
state->mode=ERROR; |
| 776 |
return; |
| 777 |
} |
| 778 |
break; |
| 779 |
case OPEN_FILE: |
| 780 |
case INITIAL_WRITE: |
| 781 |
if (!NT_STATUS_IS_OK(benchrw_open(tctx, req,state))){ |
| 782 |
torture_comment(tctx, "Failed to open/write the " |
| 783 |
"file - %s\n", |
| 784 |
nt_errstr(req->status)); |
| 785 |
state->mode=ERROR; |
| 786 |
state->readcnt=0; |
| 787 |
return; |
| 788 |
} |
| 789 |
break; |
| 790 |
case READ_WRITE_DATA: |
| 791 |
while (state->num_parallel_requests |
| 792 |
< state->lpcfg_params->num_parallel_requests) { |
| 793 |
NTSTATUS status; |
| 794 |
status = benchrw_readwrite(tctx,state); |
| 795 |
if (!NT_STATUS_IS_OK(status)){ |
| 796 |
torture_comment(tctx, "Failed to read/write " |
| 797 |
"the file - %s\n", |
| 798 |
nt_errstr(req->status)); |
| 799 |
state->mode=ERROR; |
| 800 |
return; |
| 801 |
} |
| 802 |
} |
| 803 |
break; |
| 804 |
case MAX_OPS_REACHED: |
| 805 |
if (!NT_STATUS_IS_OK(benchrw_close(tctx,req,state))){ |
| 806 |
torture_comment(tctx, "Failed to read/write/close " |
| 807 |
"the file - %s\n", |
| 808 |
nt_errstr(req->status)); |
| 809 |
state->mode=ERROR; |
| 810 |
return; |
| 811 |
} |
| 812 |
break; |
| 813 |
case CLOSE_FILE: |
| 814 |
torture_comment(tctx, "File %d closed\n",state->nr); |
| 815 |
if (!NT_STATUS_IS_OK(req->status)) { |
| 816 |
torture_comment(tctx, "Failed to close the " |
| 817 |
"file - %s\n", |
| 818 |
nt_errstr(req->status)); |
| 819 |
state->mode=ERROR; |
| 820 |
return; |
| 821 |
} |
| 822 |
state->mode=CLEANUP; |
| 823 |
return; |
| 824 |
default: |
| 825 |
break; |
| 826 |
} |
| 827 |
|
| 828 |
} |
| 829 |
|
| 830 |
/* open connection async callback function*/ |
| 831 |
static void async_open_callback(struct composite_context *con) |
| 832 |
{ |
| 833 |
struct benchrw_state *state = con->async.private_data; |
| 834 |
struct torture_context *tctx = state->tctx; |
| 835 |
int retry = state->lpcfg_params->retry; |
| 836 |
|
| 837 |
if (NT_STATUS_IS_OK(con->status)) { |
| 838 |
state->cli=((struct smb_composite_connect*) |
| 839 |
state->req_params)->out.tree; |
| 840 |
state->mode=CLEANUP_TESTDIR; |
| 841 |
}else{ |
| 842 |
if(state->writecnt < retry){ |
| 843 |
torture_comment(tctx, "Failed to open connection: " |
| 844 |
"%d, Retry (%d/%d)\n", |
| 845 |
state->nr,state->writecnt,retry); |
| 846 |
state->writecnt++; |
| 847 |
state->mode=START; |
| 848 |
usleep(1000); |
| 849 |
}else{ |
| 850 |
torture_comment(tctx, "Failed to open connection " |
| 851 |
"(%d) - %s\n", |
| 852 |
state->nr, nt_errstr(con->status)); |
| 853 |
state->mode=ERROR; |
| 854 |
} |
| 855 |
return; |
| 856 |
} |
| 857 |
} |
| 858 |
|
| 859 |
/* |
| 860 |
establishes a smbcli_tree from scratch (async) |
| 861 |
*/ |
| 862 |
static struct composite_context *torture_connect_async( |
| 863 |
struct torture_context *tctx, |
| 864 |
struct smb_composite_connect *smb, |
| 865 |
TALLOC_CTX *mem_ctx, |
| 866 |
struct tevent_context *ev, |
| 867 |
const char *host, |
| 868 |
const char *share, |
| 869 |
const char *workgroup) |
| 870 |
{ |
| 871 |
torture_comment(tctx, "Open Connection to %s/%s\n",host,share); |
| 872 |
smb->in.dest_host=talloc_strdup(mem_ctx,host); |
| 873 |
smb->in.service=talloc_strdup(mem_ctx,share); |
| 874 |
smb->in.socket_options = lpcfg_socket_options(tctx->lp_ctx); |
| 875 |
smb->in.called_name = strupper_talloc(mem_ctx, host); |
| 876 |
smb->in.service_type=NULL; |
| 877 |
smb->in.credentials = samba_cmdline_get_creds(); |
| 878 |
smb->in.fallback_to_anonymous=false; |
| 879 |
smb->in.gensec_settings = lpcfg_gensec_settings(mem_ctx, tctx->lp_ctx); |
| 880 |
smb->in.workgroup=workgroup; |
| 881 |
lpcfg_smbcli_options(tctx->lp_ctx, &smb->in.options); |
| 882 |
lpcfg_smbcli_session_options(tctx->lp_ctx, &smb->in.session_options); |
| 883 |
|
| 884 |
return smb_composite_connect_send(smb,mem_ctx, tctx->lp_ctx, |
| 885 |
lpcfg_resolve_context(tctx->lp_ctx),ev); |
| 886 |
} |
| 887 |
|
| 888 |
bool run_benchrw(struct torture_context *tctx) |
| 889 |
{ |
| 890 |
struct smb_composite_connect *smb_con; |
| 891 |
const char *fname = "\\rwtest.dat"; |
| 892 |
struct smbcli_request *req; |
| 893 |
struct benchrw_state **state; |
| 894 |
int i , num_unc_names; |
| 895 |
struct tevent_context *ev ; |
| 896 |
struct composite_context *req1; |
| 897 |
struct bench_params lpparams; |
| 898 |
union smb_mkdir parms; |
| 899 |
int finished = 0; |
| 900 |
bool success=true; |
| 901 |
int torture_nprocs = torture_setting_int(tctx, "nprocs", 4); |
| 902 |
|
| 903 |
torture_comment(tctx, "Start BENCH-READWRITE num_ops=%d " |
| 904 |
"num_nprocs=%d\n", |
| 905 |
torture_numops, torture_nprocs); |
| 906 |
|
| 907 |
/*init talloc context*/ |
| 908 |
ev = tctx->ev; |
| 909 |
state = talloc_array(tctx, struct benchrw_state *, torture_nprocs); |
| 910 |
|
| 911 |
/* init params using lpcfg_parm_xxx */ |
| 912 |
num_unc_names = init_benchrw_params(tctx,&lpparams); |
| 913 |
|
| 914 |
/* init private data structs*/ |
| 915 |
for(i = 0; i<torture_nprocs;i++){ |
| 916 |
state[i]=talloc(tctx,struct benchrw_state); |
| 917 |
state[i]->tctx = tctx; |
| 918 |
state[i]->completed=0; |
| 919 |
state[i]->num_parallel_requests=0; |
| 920 |
state[i]->lpcfg_params=&lpparams; |
| 921 |
state[i]->nr=i; |
| 922 |
state[i]->dname=talloc_asprintf(tctx,"benchrw%d",i); |
| 923 |
state[i]->fname=talloc_asprintf(tctx,"%s%s", |
| 924 |
state[i]->dname,fname); |
| 925 |
state[i]->mode=START; |
| 926 |
state[i]->writecnt=0; |
| 927 |
} |
| 928 |
|
| 929 |
torture_comment(tctx, "Starting async requests\n"); |
| 930 |
while(finished != torture_nprocs){ |
| 931 |
finished=0; |
| 932 |
for(i = 0; i<torture_nprocs;i++){ |
| 933 |
switch (state[i]->mode){ |
| 934 |
/*open multiple connections with the same userid */ |
| 935 |
case START: |
| 936 |
smb_con = talloc_zero( |
| 937 |
tctx,struct smb_composite_connect); |
| 938 |
state[i]->req_params=smb_con; |
| 939 |
state[i]->mode=OPEN_CONNECTION; |
| 940 |
req1 = torture_connect_async( |
| 941 |
tctx, smb_con, tctx,ev, |
| 942 |
lpparams.unc[i % num_unc_names]->host, |
| 943 |
lpparams.unc[i % num_unc_names]->share, |
| 944 |
lpparams.workgroup); |
| 945 |
/* register callback fn + private data */ |
| 946 |
req1->async.fn = async_open_callback; |
| 947 |
req1->async.private_data=state[i]; |
| 948 |
break; |
| 949 |
/*setup test dirs (sync)*/ |
| 950 |
case CLEANUP_TESTDIR: |
| 951 |
torture_comment(tctx, "Setup test dir %d\n",i); |
| 952 |
smb_raw_exit(state[i]->cli->session); |
| 953 |
if (smbcli_deltree(state[i]->cli, |
| 954 |
state[i]->dname) == -1) { |
| 955 |
torture_comment( |
| 956 |
tctx, |
| 957 |
"Unable to delete %s - %s\n", |
| 958 |
state[i]->dname, |
| 959 |
smbcli_errstr(state[i]->cli)); |
| 960 |
state[i]->mode=ERROR; |
| 961 |
break; |
| 962 |
} |
| 963 |
state[i]->mode=MK_TESTDIR; |
| 964 |
parms.mkdir.level = RAW_MKDIR_MKDIR; |
| 965 |
parms.mkdir.in.path = state[i]->dname; |
| 966 |
req = smb_raw_mkdir_send(state[i]->cli,&parms); |
| 967 |
/* register callback fn + private data */ |
| 968 |
req->async.fn = benchrw_callback; |
| 969 |
req->async.private_data=state[i]; |
| 970 |
break; |
| 971 |
/* error occurred , finish */ |
| 972 |
case ERROR: |
| 973 |
finished++; |
| 974 |
success=false; |
| 975 |
break; |
| 976 |
/* cleanup , close connection */ |
| 977 |
case CLEANUP: |
| 978 |
torture_comment(tctx, "Deleting test dir %s " |
| 979 |
"%d/%d\n",state[i]->dname, |
| 980 |
i+1,torture_nprocs); |
| 981 |
smbcli_deltree(state[i]->cli,state[i]->dname); |
| 982 |
if (NT_STATUS_IS_ERR(smb_tree_disconnect( |
| 983 |
state[i]->cli))) { |
| 984 |
torture_comment(tctx, "ERROR: Tree " |
| 985 |
"disconnect failed"); |
| 986 |
state[i]->mode=ERROR; |
| 987 |
break; |
| 988 |
} |
| 989 |
state[i]->mode=FINISHED; |
| 990 |
|
| 991 |
FALL_THROUGH; |
| 992 |
case FINISHED: |
| 993 |
finished++; |
| 994 |
break; |
| 995 |
default: |
| 996 |
tevent_loop_once(ev); |
| 997 |
} |
| 998 |
} |
| 999 |
} |
| 1000 |
|
| 1001 |
return success; |
| 1002 |
} |
| 1003 |
|