Debian Codegraph @

xlators/cluster/afr/src/afr-self-heal-common.c

raw
1 /*
2 Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
3 This file is part of GlusterFS.
4
5 This file is licensed to you under your choice of the GNU Lesser
6 General Public License, version 3 or any later version (LGPLv3 or
7 later), or the GNU General Public License, version 2 (GPLv2), in all
8 cases as published by the Free Software Foundation.
9 */
10
11 #include "afr.h"
12 #include "afr-self-heal.h"
13 #include "protocol-common.h"
14 #include "afr-messages.h"
15 #include <glusterfs/events.h>
16 #include <openssl/md5.h>
17
18 void
19 afr_heal_synctask(xlator_t *this, afr_local_t *local);
20
21 int
22 afr_lookup_and_heal_gfid(xlator_t *this, inode_t *parent, const char *name,
23 inode_t *inode, struct afr_reply *replies, int source,
24 unsigned char *sources, void *gfid, int *gfid_idx)
25 {
26 afr_private_t *priv = NULL;
27 call_frame_t *frame = NULL;
28 afr_local_t *local = NULL;
29 unsigned char *wind_on = NULL;
30 ia_type_t ia_type = IA_INVAL;
31 dict_t *xdata = NULL;
32 loc_t loc = {
33 0,
34 };
35 int ret = 0;
36 int i = 0;
37
38 priv = this->private;
39 wind_on = alloca0(priv->child_count);
40 if (source >= 0 && replies[source].valid && replies[source].op_ret == 0)
41 ia_type = replies[source].poststat.ia_type;
42
43 if (ia_type != IA_INVAL)
44 goto heal;
45
46 /* If ia_type is still invalid, it means either
47 * (a)'source' was -1, i.e. parent dir pending xattrs are in split-brain
48 * (or) (b) The parent dir pending xattrs are all zeroes (i.e. all bricks
49 * are sources) and the 'source' we selected earlier might be the one where
50 * the file is not actually present.
51 *
52 * In both cases, let us pick a brick with a successful reply and use its
53 * ia_type.
54 * */
55 for (i = 0; i < priv->child_count; i++) {
56 if (source == -1) {
57 /* case (a) above. */
58 if (replies[i].valid && replies[i].op_ret == 0 &&
59 replies[i].poststat.ia_type != IA_INVAL) {
60 ia_type = replies[i].poststat.ia_type;
61 break;
62 }
63 } else {
64 /* case (b) above. */
65 if (i == source)
66 continue;
67 if (sources[i] && replies[i].valid && replies[i].op_ret == 0 &&
68 replies[i].poststat.ia_type != IA_INVAL) {
69 ia_type = replies[i].poststat.ia_type;
70 break;
71 }
72 }
73 }
74
75 heal:
76 /* gfid heal on those subvolumes that do not have gfid associated
77 * with the inode and update those replies.
78 */
79 for (i = 0; i < priv->child_count; i++) {
80 if (!replies[i].valid || replies[i].op_ret != 0)
81 continue;
82
83 if (gf_uuid_is_null(gfid) &&
84 !gf_uuid_is_null(replies[i].poststat.ia_gfid) &&
85 replies[i].poststat.ia_type == ia_type)
86 gfid = replies[i].poststat.ia_gfid;
87
88 if (!gf_uuid_is_null(replies[i].poststat.ia_gfid) ||
89 replies[i].poststat.ia_type != ia_type)
90 continue;
91
92 wind_on[i] = 1;
93 }
94
95 if (AFR_COUNT(wind_on, priv->child_count) == 0)
96 return 0;
97
98 xdata = dict_new();
99 if (!xdata) {
100 ret = -ENOMEM;
101 goto out;
102 }
103
104 ret = dict_set_gfuuid(xdata, "gfid-req", gfid, true);
105 if (ret) {
106 ret = -ENOMEM;
107 goto out;
108 }
109
110 frame = afr_frame_create(this, &ret);
111 if (!frame) {
112 ret = -ret;
113 goto out;
114 }
115
116 local = frame->local;
117 loc.parent = inode_ref(parent);
118 gf_uuid_copy(loc.pargfid, parent->gfid);
119 loc.name = name;
120 loc.inode = inode_ref(inode);
121
122 AFR_ONLIST(wind_on, frame, afr_selfheal_discover_cbk, lookup, &loc, xdata);
123
124 for (i = 0; i < priv->child_count; i++) {
125 if (!wind_on[i])
126 continue;
127 afr_reply_wipe(&replies[i]);
128 afr_reply_copy(&replies[i], &local->replies[i]);
129 }
130 if (gfid_idx && (*gfid_idx == -1)) {
131 /*Pick a brick where the gifd heal was successful.*/
132 for (i = 0; i < priv->child_count; i++) {
133 if (!wind_on[i])
134 continue;
135 if (replies[i].valid && replies[i].op_ret == 0 &&
136 !gf_uuid_is_null(replies[i].poststat.ia_gfid)) {
137 *gfid_idx = i;
138 break;
139 }
140 }
141 }
142 out:
143 if (gfid_idx && (*gfid_idx == -1) && (ret == 0) && local) {
144 ret = -afr_final_errno(local, priv);
145 }
146 loc_wipe(&loc);
147 if (frame)
148 AFR_STACK_DESTROY(frame);
149 if (xdata)
150 dict_unref(xdata);
151
152 return ret;
153 }
154
155 static int
156 afr_gfid_sbrain_source_from_src_brick(xlator_t *this, struct afr_reply *replies,
157 char *src_brick)
158 {
159 int i = 0;
160 afr_private_t *priv = NULL;
161
162 priv = this->private;
163 for (i = 0; i < priv->child_count; i++) {
164 if (!replies[i].valid || replies[i].op_ret == -1)
165 continue;
166 if (strcmp(priv->children[i]->name, src_brick) == 0)
167 return i;
168 }
169 return -1;
170 }
171
172 static int
173 afr_selfheal_gfid_mismatch_by_majority(struct afr_reply *replies,
174 int child_count)
175 {
176 int j = 0;
177 int i = 0;
178 int votes;
179
180 for (i = 0; i < child_count; i++) {
181 if (!replies[i].valid || replies[i].op_ret == -1)
182 continue;
183
184 votes = 1;
185 for (j = i + 1; j < child_count; j++) {
186 if ((!gf_uuid_compare(replies[i].poststat.ia_gfid,
187 replies[j].poststat.ia_gfid)))
188 votes++;
189 if (votes > child_count / 2)
190 return i;
191 }
192 }
193
194 return -1;
195 }
196
197 static int
198 afr_gfid_sbrain_source_from_bigger_file(struct afr_reply *replies,
199 int child_count)
200 {
201 int i = 0;
202 int src = -1;
203 uint64_t size = 0;
204
205 for (i = 0; i < child_count; i++) {
206 if (!replies[i].valid || replies[i].op_ret == -1)
207 continue;
208 if (size < replies[i].poststat.ia_size) {
209 src = i;
210 size = replies[i].poststat.ia_size;
211 } else if (replies[i].poststat.ia_size == size) {
212 src = -1;
213 }
214 }
215 return src;
216 }
217
218 static int
219 afr_gfid_sbrain_source_from_latest_mtime(struct afr_reply *replies,
220 int child_count)
221 {
222 int i = 0;
223 int src = -1;
224 uint64_t mtime = 0;
225 uint32_t mtime_nsec = 0;
226
227 for (i = 0; i < child_count; i++) {
228 if (!replies[i].valid || replies[i].op_ret != 0)
229 continue;
230 if ((mtime < replies[i].poststat.ia_mtime) ||
231 ((mtime == replies[i].poststat.ia_mtime) &&
232 (mtime_nsec < replies[i].poststat.ia_mtime_nsec))) {
233 src = i;
234 mtime = replies[i].poststat.ia_mtime;
235 mtime_nsec = replies[i].poststat.ia_mtime_nsec;
236 } else if ((mtime == replies[i].poststat.ia_mtime) &&
237 (mtime_nsec == replies[i].poststat.ia_mtime_nsec)) {
238 src = -1;
239 }
240 }
241 return src;
242 }
243
244 int
245 afr_gfid_split_brain_source(xlator_t *this, struct afr_reply *replies,
246 inode_t *inode, uuid_t pargfid, const char *bname,
247 int src_idx, int child_idx,
248 unsigned char *locked_on, int *src, dict_t *req,
249 dict_t *rsp)
250 {
251 afr_private_t *priv = NULL;
252 char g1[64] = {
253 0,
254 };
255 char g2[64] = {
256 0,
257 };
258 int up_count = 0;
259 int heal_op = -1;
260 int ret = -1;
261 char *src_brick = NULL;
262
263 *src = -1;
264 priv = this->private;
265 up_count = AFR_COUNT(locked_on, priv->child_count);
266 if (up_count != priv->child_count) {
267 gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
268 "All the bricks should be up to resolve the gfid split "
269 "barin");
270 if (rsp) {
271 ret = dict_set_sizen_str_sizen(rsp, "gfid-heal-msg",
272 SALL_BRICKS_UP_TO_RESOLVE);
273 if (ret)
274 gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_DICT_SET_FAILED,
275 "Error setting"
276 " gfid-heal-msg dict");
277 }
278 goto out;
279 }
280
281 if (req) {
282 ret = dict_get_int32_sizen(req, "heal-op", &heal_op);
283 if (ret)
284 goto fav_child;
285 } else {
286 goto fav_child;
287 }
288
289 switch (heal_op) {
290 case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
291 *src = afr_gfid_sbrain_source_from_bigger_file(replies,
292 priv->child_count);
293 if (*src == -1) {
294 gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
295 SNO_BIGGER_FILE);
296 if (rsp) {
297 ret = dict_set_sizen_str_sizen(rsp, "gfid-heal-msg",
298 SNO_BIGGER_FILE);
299 if (ret)
300 gf_msg(this->name, GF_LOG_ERROR, 0,
301 AFR_MSG_DICT_SET_FAILED,
302 "Error"
303 " setting gfid-heal-msg dict");
304 }
305 }
306 break;
307
308 case GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME:
309 *src = afr_gfid_sbrain_source_from_latest_mtime(replies,
310 priv->child_count);
311 if (*src == -1) {
312 gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
313 SNO_DIFF_IN_MTIME);
314 if (rsp) {
315 ret = dict_set_sizen_str_sizen(rsp, "gfid-heal-msg",
316 SNO_DIFF_IN_MTIME);
317 if (ret)
318 gf_msg(this->name, GF_LOG_ERROR, 0,
319 AFR_MSG_DICT_SET_FAILED,
320 "Error"
321 "setting gfid-heal-msg dict");
322 }
323 }
324 break;
325
326 case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
327 ret = dict_get_str_sizen(req, "child-name", &src_brick);
328 if (ret) {
329 gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
330 "Error getting the source "
331 "brick");
332 break;
333 }
334 *src = afr_gfid_sbrain_source_from_src_brick(this, replies,
335 src_brick);
336 if (*src == -1) {
337 gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
338 SERROR_GETTING_SRC_BRICK);
339 if (rsp) {
340 ret = dict_set_sizen_str_sizen(rsp, "gfid-heal-msg",
341 SERROR_GETTING_SRC_BRICK);
342 if (ret)
343 gf_msg(this->name, GF_LOG_ERROR, 0,
344 AFR_MSG_DICT_SET_FAILED,
345 "Error"
346 " setting gfid-heal-msg dict");
347 }
348 }
349 break;
350
351 default:
352 break;
353 }
354 goto out;
355
356 fav_child:
357 switch (priv->fav_child_policy) {
358 case AFR_FAV_CHILD_BY_SIZE:
359 *src = afr_sh_fav_by_size(this, replies, inode);
360 break;
361 case AFR_FAV_CHILD_BY_MTIME:
362 *src = afr_sh_fav_by_mtime(this, replies, inode);
363 break;
364 case AFR_FAV_CHILD_BY_CTIME:
365 *src = afr_sh_fav_by_ctime(this, replies, inode);
366 break;
367 case AFR_FAV_CHILD_BY_MAJORITY:
368 if (priv->child_count != 2)
369 *src = afr_selfheal_gfid_mismatch_by_majority(
370 replies, priv->child_count);
371 else
372 *src = -1;
373
374 if (*src == -1) {
375 gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
376 "No majority to resolve "
377 "gfid split brain");
378 }
379 break;
380 default:
381 break;
382 }
383 /* At this point we have a source selected by favourite child policy,
384 * If this is for a directory it might not be a good idea to automatically
385 * resolve the GFID. Because the ultimate view of a dir is owned by DHT.
386 * But if it is not a distributed volume, then it might be fine to do
387 * the automatic gfid fix.
388 * So here we skip the automatic split brain resolution
389 */
390 if (*src != -1 && IA_ISDIR(replies[*src].poststat.ia_type)) {
391 gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SPLIT_BRAIN,
392 "Automatic Gfid mismatch resolution for directories will be "
393 "skipped. Please manually resolve the splitbrain using cli");
394 *src = -1;
395 }
396
397 out:
398 if (*src == -1) {
399 gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
400 "Gfid mismatch detected for <gfid:%s>/%s>, %s on %s and"
401 " %s on %s.",
402 uuid_utoa(pargfid), bname,
403 uuid_utoa_r(replies[child_idx].poststat.ia_gfid, g1),
404 priv->children[child_idx]->name,
405 uuid_utoa_r(replies[src_idx].poststat.ia_gfid, g2),
406 priv->children[src_idx]->name);
407 gf_event(EVENT_AFR_SPLIT_BRAIN,
408 "client-pid=%d;"
409 "subvol=%s;type=gfid;file="
410 "<gfid:%s>/%s>;count=2;child-%d=%s;gfid-%d=%s;"
411 "child-%d=%s;gfid-%d=%s",
412 this->ctx->cmd_args.client_pid, this->name, uuid_utoa(pargfid),
413 bname, child_idx, priv->children[child_idx]->name, child_idx,
414 uuid_utoa_r(replies[child_idx].poststat.ia_gfid, g1), src_idx,
415 priv->children[src_idx]->name, src_idx,
416 uuid_utoa_r(replies[src_idx].poststat.ia_gfid, g2));
417 return -EIO;
418 }
419 return 0;
420 }
421
422 static int
423 afr_selfheal_post_op_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
424 int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
425 {
426 afr_local_t *local = NULL;
427
428 local = frame->local;
429
430 local->op_ret = op_ret;
431 local->op_errno = op_errno;
432 syncbarrier_wake(&local->barrier);
433
434 return 0;
435 }
436
437 int
438 afr_selfheal_post_op(call_frame_t *frame, xlator_t *this, inode_t *inode,
439 int subvol, dict_t *xattr, dict_t *xdata)
440 {
441 afr_private_t *priv = NULL;
442 afr_local_t *local = NULL;
443 loc_t loc = {
444 0,
445 };
446 int ret = 0;
447
448 priv = this->private;
449 local = frame->local;
450
451 loc.inode = inode_ref(inode);
452 gf_uuid_copy(loc.gfid, inode->gfid);
453
454 local->op_ret = 0;
455
456 STACK_WIND(frame, afr_selfheal_post_op_cbk, priv->children[subvol],
457 priv->children[subvol]->fops->xattrop, &loc,
458 GF_XATTROP_ADD_ARRAY, xattr, xdata);
459
460 syncbarrier_wait(&local->barrier, 1);
461 if (local->op_ret < 0)
462 ret = -local->op_errno;
463
464 loc_wipe(&loc);
465 local->op_ret = 0;
466
467 return ret;
468 }
469
470 static int
471 afr_check_stale_error(struct afr_reply *replies, afr_private_t *priv)
472 {
473 int i = 0;
474 int op_errno = 0;
475 int tmp_errno = 0;
476 int stale_count = 0;
477
478 for (i = 0; i < priv->child_count; i++) {
479 tmp_errno = replies[i].op_errno;
480 if (tmp_errno == ENOENT || tmp_errno == ESTALE) {
481 op_errno = afr_higher_errno(op_errno, tmp_errno);
482 stale_count++;
483 }
484 }
485 if (stale_count != priv->child_count)
486 return -ENOTCONN;
487 else
488 return -op_errno;
489 }
490
491 int
492 afr_sh_generic_fop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
493 int op_ret, int op_errno, struct iatt *pre,
494 struct iatt *post, dict_t *xdata)
495 {
496 int i = (long)cookie;
497 afr_local_t *local = NULL;
498
499 local = frame->local;
500
501 local->replies[i].valid = 1;
502 local->replies[i].op_ret = op_ret;
503 local->replies[i].op_errno = op_errno;
504 if (pre)
505 local->replies[i].prestat = *pre;
506 if (post)
507 local->replies[i].poststat = *post;
508 if (xdata)
509 local->replies[i].xdata = dict_ref(xdata);
510
511 syncbarrier_wake(&local->barrier);
512
513 return 0;
514 }
515
516 int
517 afr_selfheal_restore_time(call_frame_t *frame, xlator_t *this, inode_t *inode,
518 int source, unsigned char *healed_sinks,
519 struct afr_reply *replies)
520 {
521 loc_t loc = {
522 0,
523 };
524
525 loc.inode = inode_ref(inode);
526 gf_uuid_copy(loc.gfid, inode->gfid);
527
528 AFR_ONLIST(healed_sinks, frame, afr_sh_generic_fop_cbk, setattr, &loc,
529 &replies[source].poststat,
530 (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME | GF_SET_ATTR_CTIME),
531 NULL);
532
533 loc_wipe(&loc);
534
535 return 0;
536 }
537
538 static dict_t *
539 afr_selfheal_output_xattr(xlator_t *this, gf_boolean_t is_full_crawl,
540 afr_transaction_type type, int *output_dirty,
541 int **output_matrix, int subvol,
542 int **full_heal_mtx_out)
543 {
544 int j = 0;
545 int idx = 0;
546 int d_idx = 0;
547 int ret = 0;
548 int *raw = 0;
549 dict_t *xattr = NULL;
550 afr_private_t *priv = NULL;
551
552 priv = this->private;
553 idx = afr_index_for_transaction_type(type);
554 d_idx = afr_index_for_transaction_type(AFR_DATA_TRANSACTION);
555
556 xattr = dict_new();
557 if (!xattr)
558 return NULL;
559
560 /* clear dirty */
561 raw = GF_CALLOC(sizeof(int), AFR_NUM_CHANGE_LOGS, gf_afr_mt_int32_t);
562 if (!raw)
563 goto err;
564
565 raw[idx] = htobe32(output_dirty[subvol]);
566 ret = dict_set_bin(xattr, AFR_DIRTY, raw,
567 sizeof(int) * AFR_NUM_CHANGE_LOGS);
568 if (ret) {
569 GF_FREE(raw);
570 goto err;
571 }
572
573 /* clear/set pending */
574 for (j = 0; j < priv->child_count; j++) {
575 raw = GF_CALLOC(sizeof(int), AFR_NUM_CHANGE_LOGS, gf_afr_mt_int32_t);
576 if (!raw)
577 goto err;
578
579 raw[idx] = htobe32(output_matrix[subvol][j]);
580 if (is_full_crawl)
581 raw[d_idx] = htobe32(full_heal_mtx_out[subvol][j]);
582
583 ret = dict_set_bin(xattr, priv->pending_key[j], raw,
584 sizeof(int) * AFR_NUM_CHANGE_LOGS);
585 if (ret) {
586 GF_FREE(raw);
587 goto err;
588 }
589 }
590
591 return xattr;
592 err:
593 if (xattr)
594 dict_unref(xattr);
595 return NULL;
596 }
597
598 int
599 afr_selfheal_undo_pending(call_frame_t *frame, xlator_t *this, inode_t *inode,
600 unsigned char *sources, unsigned char *sinks,
601 unsigned char *healed_sinks,
602 unsigned char *undid_pending,
603 afr_transaction_type type, struct afr_reply *replies,
604 unsigned char *locked_on)
605 {
606 afr_private_t *priv = NULL;
607 afr_local_t *local = NULL;
608 int i = 0;
609 int j = 0;
610 unsigned char *pending = NULL;
611 int *input_dirty = NULL;
612 int **input_matrix = NULL;
613 int **full_heal_mtx_in = NULL;
614 int **full_heal_mtx_out = NULL;
615 int *output_dirty = NULL;
616 int **output_matrix = NULL;
617 dict_t *xattr = NULL;
618 dict_t *xdata = NULL;
619
620 priv = this->private;
621 local = frame->local;
622
623 pending = alloca0(priv->child_count);
624
625 input_dirty = alloca0(priv->child_count * sizeof(int));
626 input_matrix = ALLOC_MATRIX(priv->child_count, int);
627 full_heal_mtx_in = ALLOC_MATRIX(priv->child_count, int);
628 full_heal_mtx_out = ALLOC_MATRIX(priv->child_count, int);
629 output_dirty = alloca0(priv->child_count * sizeof(int));
630 output_matrix = ALLOC_MATRIX(priv->child_count, int);
631
632 xdata = dict_new();
633 if (!xdata)
634 return -1;
635
636 afr_selfheal_extract_xattr(this, replies, type, input_dirty, input_matrix);
637
638 if (local->need_full_crawl)
639 afr_selfheal_extract_xattr(this, replies, AFR_DATA_TRANSACTION, NULL,
640 full_heal_mtx_in);
641
642 for (i = 0; i < priv->child_count; i++)
643 if (sinks[i] && !healed_sinks[i])
644 pending[i] = 1;
645
646 for (i = 0; i < priv->child_count; i++) {
647 for (j = 0; j < priv->child_count; j++) {
648 if (pending[j]) {
649 output_matrix[i][j] = 1;
650 if (type == AFR_ENTRY_TRANSACTION)
651 full_heal_mtx_out[i][j] = 1;
652 } else if (locked_on[j]) {
653 output_matrix[i][j] = -input_matrix[i][j];
654 if (type == AFR_ENTRY_TRANSACTION)
655 full_heal_mtx_out[i][j] = -full_heal_mtx_in[i][j];
656 }
657 }
658 }
659
660 for (i = 0; i < priv->child_count; i++) {
661 if (!pending[i])
662 output_dirty[i] = -input_dirty[i];
663 }
664
665 for (i = 0; i < priv->child_count; i++) {
666 if (!locked_on[i])
667 /* perform post-op only on subvols we had locked
668 and inspected on.
669 */
670 continue;
671 if (undid_pending[i])
672 /* We already unset the pending xattrs in
673 * _afr_fav_child_reset_sink_xattrs(). */
674 continue;
675
676 xattr = afr_selfheal_output_xattr(this, local->need_full_crawl, type,
677 output_dirty, output_matrix, i,
678 full_heal_mtx_out);
679 if (!xattr) {
680 continue;
681 }
682
683 if ((type == AFR_ENTRY_TRANSACTION) && (priv->esh_granular)) {
684 if (xdata && dict_set_int8(xdata, GF_XATTROP_PURGE_INDEX, 1))
685 gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_DICT_SET_FAILED,
686 "Failed to set"
687 " dict value for %s",
688 GF_XATTROP_PURGE_INDEX);
689 }
690
691 afr_selfheal_post_op(frame, this, inode, i, xattr, xdata);
692 dict_unref(xattr);
693 }
694
695 if (xdata)
696 dict_unref(xdata);
697
698 return 0;
699 }
700
701 void
702 afr_reply_copy(struct afr_reply *dst, struct afr_reply *src)
703 {
704 dict_t *xdata = NULL;
705
706 dst->valid = src->valid;
707 dst->op_ret = src->op_ret;
708 dst->op_errno = src->op_errno;
709 dst->prestat = src->prestat;
710 dst->poststat = src->poststat;
711 dst->preparent = src->preparent;
712 dst->postparent = src->postparent;
713 dst->preparent2 = src->preparent2;
714 dst->postparent2 = src->postparent2;
715 if (src->xdata)
716 xdata = dict_ref(src->xdata);
717 else
718 xdata = NULL;
719 if (dst->xdata)
720 dict_unref(dst->xdata);
721 dst->xdata = xdata;
722 if (xdata && dict_get_str_boolean(xdata, "fips-mode-rchecksum",
723 _gf_false) == _gf_true) {
724 memcpy(dst->checksum, src->checksum, SHA256_DIGEST_LENGTH);
725 } else {
726 memcpy(dst->checksum, src->checksum, MD5_DIGEST_LENGTH);
727 }
728 dst->fips_mode_rchecksum = src->fips_mode_rchecksum;
729 }
730
731 void
732 afr_replies_copy(struct afr_reply *dst, struct afr_reply *src, int count)
733 {
734 int i = 0;
735
736 if (dst == src)
737 return;
738
739 for (i = 0; i < count; i++) {
740 afr_reply_copy(&dst[i], &src[i]);
741 }
742 }
743
744 static int
745 afr_selfheal_fill_dirty(xlator_t *this, int *dirty, int subvol, int idx,
746 dict_t *xdata)
747 {
748 void *pending_raw = NULL;
749
750 if (!dirty)
751 return 0;
752
753 if (dict_get_ptr(xdata, AFR_DIRTY, &pending_raw))
754 return -1;
755
756 if (!pending_raw)
757 return -1;
758
759 dirty[subvol] = be32toh(*((int *)pending_raw + idx));
760
761 return 0;
762 }
763
764 void
765 afr_selfheal_fill_cell(afr_private_t *priv, dict_t *src_xdata, int *cell,
766 int sink, int idx)
767 {
768 void *pending_raw = NULL;
769
770 *cell = 0;
771 if (dict_get_ptr(src_xdata, priv->pending_key[sink], &pending_raw))
772 goto out;
773
774 if (!pending_raw)
775 goto out;
776
777 *cell = be32toh(*((int *)pending_raw + idx));
778 out:
779 return;
780 }
781
782 int
783 afr_selfheal_fill_matrix(xlator_t *this, int **matrix, int subvol, int idx,
784 dict_t *xdata)
785 {
786 int i = 0;
787 afr_private_t *priv = NULL;
788
789 priv = this->private;
790
791 if (!matrix)
792 return 0;
793
794 for (i = 0; i < priv->child_count; i++) {
795 afr_selfheal_fill_cell(priv, xdata, &matrix[subvol][i], i, idx);
796 }
797
798 return 0;
799 }
800
801 int
802 afr_selfheal_extract_xattr(xlator_t *this, struct afr_reply *replies,
803 afr_transaction_type type, int *dirty, int **matrix)
804 {
805 afr_private_t *priv = NULL;
806 int i = 0;
807 dict_t *xdata = NULL;
808 int idx = -1;
809
810 idx = afr_index_for_transaction_type(type);
811
812 priv = this->private;
813
814 for (i = 0; i < priv->child_count; i++) {
815 if (!replies[i].valid || replies[i].op_ret != 0)
816 continue;
817
818 if (!replies[i].xdata)
819 continue;
820
821 xdata = replies[i].xdata;
822
823 afr_selfheal_fill_dirty(this, dirty, i, idx, xdata);
824 afr_selfheal_fill_matrix(this, matrix, i, idx, xdata);
825 }
826
827 return 0;
828 }
829
830 /*
831 * If by chance there are multiple sources with differing sizes, select
832 * the largest file as the source.
833 *
834 * This can happen if data was directly modified in the backend or for snapshots
835 */
836 void
837 afr_mark_largest_file_as_source(xlator_t *this, unsigned char *sources,
838 struct afr_reply *replies)
839 {
840 int i = 0;
841 afr_private_t *priv = NULL;
842 uint64_t size = 0;
843
844 /* Find source with biggest file size */
845 priv = this->private;
846 for (i = 0; i < priv->child_count; i++) {
847 if (!sources[i])
848 continue;
849 if (!replies[i].valid || replies[i].op_ret != 0) {
850 sources[i] = 0;
851 continue;
852 }
853 if (size <= replies[i].poststat.ia_size) {
854 size = replies[i].poststat.ia_size;
855 }
856 }
857
858 /* Mark sources with less size as not source */
859 for (i = 0; i < priv->child_count; i++) {
860 if (!sources[i])
861 continue;
862 if (size > replies[i].poststat.ia_size)
863 sources[i] = 0;
864 }
865 }
866
867 void
868 afr_mark_latest_mtime_file_as_source(xlator_t *this, unsigned char *sources,
869 struct afr_reply *replies)
870 {
871 int i = 0;
872 afr_private_t *priv = NULL;
873 uint64_t mtime = 0;
874 uint32_t mtime_nsec = 0;
875
876 priv = this->private;
877 for (i = 0; i < priv->child_count; i++) {
878 if (!sources[i])
879 continue;
880 if (!replies[i].valid || replies[i].op_ret != 0) {
881 sources[i] = 0;
882 continue;
883 }
884 if ((mtime < replies[i].poststat.ia_mtime) ||
885 ((mtime == replies[i].poststat.ia_mtime) &&
886 (mtime_nsec < replies[i].poststat.ia_mtime_nsec))) {
887 mtime = replies[i].poststat.ia_mtime;
888 mtime_nsec = replies[i].poststat.ia_mtime_nsec;
889 }
890 }
891 for (i = 0; i < priv->child_count; i++) {
892 if (!sources[i])
893 continue;
894 if ((mtime > replies[i].poststat.ia_mtime) ||
895 ((mtime == replies[i].poststat.ia_mtime) &&
896 (mtime_nsec > replies[i].poststat.ia_mtime_nsec))) {
897 sources[i] = 0;
898 }
899 }
900 }
901
902 void
903 afr_mark_active_sinks(xlator_t *this, unsigned char *sources,
904 unsigned char *locked_on, unsigned char *sinks)
905 {
906 int i = 0;
907 afr_private_t *priv = NULL;
908
909 priv = this->private;
910
911 for (i = 0; i < priv->child_count; i++) {
912 if (!sources[i] && locked_on[i])
913 sinks[i] = 1;
914 else
915 sinks[i] = 0;
916 }
917 }
918
919 gf_boolean_t
920 afr_dict_contains_heal_op(call_frame_t *frame)
921 {
922 afr_local_t *local = NULL;
923 dict_t *xdata_req = NULL;
924 int ret = 0;
925 int heal_op = -1;
926
927 local = frame->local;
928 xdata_req = local->xdata_req;
929 ret = dict_get_int32_sizen(xdata_req, "heal-op", &heal_op);
930 if (ret)
931 return _gf_false;
932 if (local->xdata_rsp == NULL) {
933 local->xdata_rsp = dict_new();
934 if (!local->xdata_rsp)
935 return _gf_true;
936 }
937 ret = dict_set_sizen_str_sizen(local->xdata_rsp, "sh-fail-msg",
938 SFILE_NOT_IN_SPLIT_BRAIN);
939
940 return _gf_true;
941 }
942
943 gf_boolean_t
944 afr_can_decide_split_brain_source_sinks(struct afr_reply *replies,
945 int child_count)
946 {
947 int i = 0;
948
949 for (i = 0; i < child_count; i++)
950 if (replies[i].valid != 1 || replies[i].op_ret != 0)
951 return _gf_false;
952
953 return _gf_true;
954 }
955
956 int
957 afr_mark_split_brain_source_sinks_by_heal_op(
958 call_frame_t *frame, xlator_t *this, unsigned char *sources,
959 unsigned char *sinks, unsigned char *healed_sinks, unsigned char *locked_on,
960 struct afr_reply *replies, afr_transaction_type type, int heal_op)
961 {
962 afr_local_t *local = NULL;
963 afr_private_t *priv = NULL;
964 dict_t *xdata_req = NULL;
965 dict_t *xdata_rsp = NULL;
966 int ret = 0;
967 int i = 0;
968 char *name = NULL;
969 int source = -1;
970
971 local = frame->local;
972 priv = this->private;
973 xdata_req = local->xdata_req;
974
975 for (i = 0; i < priv->child_count; i++) {
976 if (locked_on[i])
977 if (sources[i] || !sinks[i] || !healed_sinks[i]) {
978 ret = -1;
979 goto out;
980 }
981 }
982 if (local->xdata_rsp == NULL) {
983 local->xdata_rsp = dict_new();
984 if (!local->xdata_rsp) {
985 ret = -1;
986 goto out;
987 }
988 }
989 xdata_rsp = local->xdata_rsp;
990
991 if (!afr_can_decide_split_brain_source_sinks(replies, priv->child_count)) {
992 ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
993 SBRAIN_HEAL_NO_GO_MSG);
994 ret = -1;
995 goto out;
996 }
997
998 for (i = 0; i < priv->child_count; i++)
999 if (locked_on[i])
1000 sources[i] = 1;
1001 switch (heal_op) {
1002 case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
1003 if (type == AFR_METADATA_TRANSACTION) {
1004 ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
1005 SUSE_SOURCE_BRICK_TO_HEAL);
1006 if (!ret)
1007 ret = -1;
1008 goto out;
1009 }
1010 afr_mark_largest_file_as_source(this, sources, replies);
1011 if (AFR_COUNT(sources, priv->child_count) != 1) {
1012 ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
1013 SNO_BIGGER_FILE);
1014 if (!ret)
1015 ret = -1;
1016 goto out;
1017 }
1018 break;
1019 case GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME:
1020 if (type == AFR_METADATA_TRANSACTION) {
1021 ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
1022 SUSE_SOURCE_BRICK_TO_HEAL);
1023 if (!ret)
1024 ret = -1;
1025 goto out;
1026 }
1027 afr_mark_latest_mtime_file_as_source(this, sources, replies);
1028 if (AFR_COUNT(sources, priv->child_count) != 1) {
1029 ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
1030 SNO_DIFF_IN_MTIME);
1031 if (!ret)
1032 ret = -1;
1033 goto out;
1034 }
1035 break;
1036 case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
1037 ret = dict_get_str_sizen(xdata_req, "child-name", &name);
1038 if (ret)
1039 goto out;
1040 source = afr_get_child_index_from_name(this, name);
1041 if (source < 0) {
1042 ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
1043 SINVALID_BRICK_NAME);
1044 if (!ret)
1045 ret = -1;
1046 goto out;
1047 }
1048 if (locked_on[source] != 1) {
1049 ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
1050 SBRICK_IS_NOT_UP);
1051 if (!ret)
1052 ret = -1;
1053 goto out;
1054 }
1055 memset(sources, 0, sizeof(*sources) * priv->child_count);
1056 sources[source] = 1;
1057 break;
1058 default:
1059 ret = -1;
1060 goto out;
1061 }
1062 for (i = 0; i < priv->child_count; i++) {
1063 if (sources[i]) {
1064 source = i;
1065 break;
1066 }
1067 }
1068 sinks[source] = 0;
1069 healed_sinks[source] = 0;
1070 ret = source;
1071 out:
1072 if (ret < 0)
1073 memset(sources, 0, sizeof(*sources) * priv->child_count);
1074 return ret;
1075 }
1076
1077 static int
1078 afr_sh_fav_by_majority(xlator_t *this, struct afr_reply *replies,
1079 inode_t *inode)
1080 {
1081 afr_private_t *priv;
1082 int vote_count = -1;
1083 int fav_child = -1;
1084 int i = 0;
1085 int k = 0;
1086
1087 priv = this->private;
1088
1089 for (i = 0; i < priv->child_count; i++) {
1090 if (replies[i].valid == 1) {
1091 gf_msg_debug(this->name, 0,
1092 "Child:%s mtime_sec = %" PRId64 ", size = %" PRIu64
1093 " for gfid %s",
1094 priv->children[i]->name, replies[i].poststat.ia_mtime,
1095 replies[i].poststat.ia_size, uuid_utoa(inode->gfid));
1096 vote_count = 0;
1097 for (k = 0; k < priv->child_count; k++) {
1098 if ((replies[k].poststat.ia_mtime ==
1099 replies[i].poststat.ia_mtime) &&
1100 (replies[k].poststat.ia_size ==
1101 replies[i].poststat.ia_size)) {
1102 vote_count++;
1103 }
1104 }
1105 if (vote_count > priv->child_count / 2) {
1106 fav_child = i;
1107 break;
1108 }
1109 }
1110 }
1111 return fav_child;
1112 }
1113
1114 /*
1115 * afr_sh_fav_by_mtime: Choose favorite child by mtime.
1116 */
1117 int
1118 afr_sh_fav_by_mtime(xlator_t *this, struct afr_reply *replies, inode_t *inode)
1119 {
1120 afr_private_t *priv;
1121 int fav_child = -1;
1122 int i = 0;
1123 uint64_t cmp_mtime = 0;
1124 uint32_t cmp_mtime_nsec = 0;
1125
1126 priv = this->private;
1127
1128 for (i = 0; i < priv->child_count; i++) {
1129 if (replies[i].valid == 1) {
1130 gf_msg_debug(this->name, 0,
1131 "Child:%s mtime = %" PRId64
1132 ", mtime_nsec = %d for "
1133 "gfid %s",
1134 priv->children[i]->name, replies[i].poststat.ia_mtime,
1135 replies[i].poststat.ia_mtime_nsec,
1136 uuid_utoa(inode->gfid));
1137 if (replies[i].poststat.ia_mtime > cmp_mtime) {
1138 cmp_mtime = replies[i].poststat.ia_mtime;
1139 cmp_mtime_nsec = replies[i].poststat.ia_mtime_nsec;
1140 fav_child = i;
1141 } else if ((replies[i].poststat.ia_mtime == cmp_mtime) &&
1142 (replies[i].poststat.ia_mtime_nsec > cmp_mtime_nsec)) {
1143 cmp_mtime = replies[i].poststat.ia_mtime;
1144 cmp_mtime_nsec = replies[i].poststat.ia_mtime_nsec;
1145 fav_child = i;
1146 }
1147 }
1148 }
1149 return fav_child;
1150 }
1151
1152 /*
1153 * afr_sh_fav_by_ctime: Choose favorite child by ctime.
1154 */
1155 int
1156 afr_sh_fav_by_ctime(xlator_t *this, struct afr_reply *replies, inode_t *inode)
1157 {
1158 afr_private_t *priv;
1159 int fav_child = -1;
1160 int i = 0;
1161 uint64_t cmp_ctime = 0;
1162 uint32_t cmp_ctime_nsec = 0;
1163
1164 priv = this->private;
1165
1166 for (i = 0; i < priv->child_count; i++) {
1167 if (replies[i].valid == 1) {
1168 gf_msg_debug(this->name, 0,
1169 "Child:%s ctime = %" PRId64
1170 ", ctime_nsec = %d for "
1171 "gfid %s",
1172 priv->children[i]->name, replies[i].poststat.ia_ctime,
1173 replies[i].poststat.ia_ctime_nsec,
1174 uuid_utoa(inode->gfid));
1175 if (replies[i].poststat.ia_ctime > cmp_ctime) {
1176 cmp_ctime = replies[i].poststat.ia_ctime;
1177 cmp_ctime_nsec = replies[i].poststat.ia_ctime_nsec;
1178 fav_child = i;
1179 } else if ((replies[i].poststat.ia_ctime == cmp_ctime) &&
1180 (replies[i].poststat.ia_ctime_nsec > cmp_ctime_nsec)) {
1181 cmp_ctime = replies[i].poststat.ia_ctime;
1182 cmp_ctime_nsec = replies[i].poststat.ia_ctime_nsec;
1183 fav_child = i;
1184 }
1185 }
1186 }
1187 return fav_child;
1188 }
1189
1190 /*
1191 * afr_sh_fav_by_size: Choose favorite child by size
1192 * when not all files are of zero size.
1193 */
1194 int
1195 afr_sh_fav_by_size(xlator_t *this, struct afr_reply *replies, inode_t *inode)
1196 {
1197 afr_private_t *priv;
1198 int fav_child = -1;
1199 int i = 0;
1200 uint64_t cmp_sz = 0;
1201
1202 priv = this->private;
1203 for (i = 0; i < priv->child_count; i++) {
1204 if (!replies[i].valid) {
1205 continue;
1206 }
1207 gf_msg_debug(this->name, 0,
1208 "Child:%s file size = %" PRIu64 " for gfid %s",
1209 priv->children[i]->name, replies[i].poststat.ia_size,
1210 uuid_utoa(inode->gfid));
1211 if (replies[i].poststat.ia_type == IA_IFDIR) {
1212 gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
1213 "Cannot perform selfheal on %s. "
1214 "Size policy is not applicable to directories.",
1215 uuid_utoa(inode->gfid));
1216 break;
1217 }
1218 if (replies[i].poststat.ia_size > cmp_sz) {
1219 cmp_sz = replies[i].poststat.ia_size;
1220 fav_child = i;
1221 } else if (replies[i].poststat.ia_size == cmp_sz) {
1222 fav_child = -1;
1223 }
1224 }
1225 if (fav_child == -1) {
1226 gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
1227 "No bigger file");
1228 }
1229 return fav_child;
1230 }
1231
1232 int
1233 afr_sh_get_fav_by_policy(xlator_t *this, struct afr_reply *replies,
1234 inode_t *inode, char **policy_str)
1235 {
1236 afr_private_t *priv = NULL;
1237 int fav_child = -1;
1238
1239 priv = this->private;
1240 if (!afr_can_decide_split_brain_source_sinks(replies, priv->child_count)) {
1241 return -1;
1242 }
1243
1244 switch (priv->fav_child_policy) {
1245 case AFR_FAV_CHILD_BY_SIZE:
1246 fav_child = afr_sh_fav_by_size(this, replies, inode);
1247 if (policy_str && fav_child >= 0) {
1248 *policy_str = "SIZE";
1249 }
1250 break;
1251 case AFR_FAV_CHILD_BY_CTIME:
1252 fav_child = afr_sh_fav_by_ctime(this, replies, inode);
1253 if (policy_str && fav_child >= 0) {
1254 *policy_str = "CTIME";
1255 }
1256 break;
1257 case AFR_FAV_CHILD_BY_MTIME:
1258 fav_child = afr_sh_fav_by_mtime(this, replies, inode);
1259 if (policy_str && fav_child >= 0) {
1260 *policy_str = "MTIME";
1261 }
1262 break;
1263 case AFR_FAV_CHILD_BY_MAJORITY:
1264 fav_child = afr_sh_fav_by_majority(this, replies, inode);
1265 if (policy_str && fav_child >= 0) {
1266 *policy_str = "MAJORITY";
1267 }
1268 break;
1269 case AFR_FAV_CHILD_NONE:
1270 default:
1271 break;
1272 }
1273
1274 return fav_child;
1275 }
1276
1277 static int
1278 afr_mark_split_brain_source_sinks_by_policy(
1279 call_frame_t *frame, xlator_t *this, inode_t *inode, unsigned char *sources,
1280 unsigned char *sinks, unsigned char *healed_sinks, unsigned char *locked_on,
1281 struct afr_reply *replies, afr_transaction_type type)
1282 {
1283 afr_private_t *priv = NULL;
1284 int fav_child = -1;
1285 char mtime_str[256];
1286 char ctime_str[256];
1287 char *policy_str = NULL;
1288 struct tm *tm_ptr;
1289 time_t time;
1290
1291 priv = this->private;
1292
1293 fav_child = afr_sh_get_fav_by_policy(this, replies, inode, &policy_str);
1294 if (fav_child == -1) {
1295 gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
1296 "No child selected by favorite-child policy.");
1297 } else if (fav_child > priv->child_count - 1) {
1298 gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
1299 "Invalid child (%d) "
1300 "selected by policy %s.",
1301 fav_child, policy_str);
1302 } else if (fav_child >= 0) {
1303 time = replies[fav_child].poststat.ia_mtime;
1304 tm_ptr = localtime(&time);
1305 strftime(mtime_str, sizeof(mtime_str), "%Y-%m-%d %H:%M:%S", tm_ptr);
1306 time = replies[fav_child].poststat.ia_ctime;
1307 tm_ptr = localtime(&time);
1308 strftime(ctime_str, sizeof(ctime_str), "%Y-%m-%d %H:%M:%S", tm_ptr);
1309
1310 gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
1311 "Source %s selected as authentic to resolve conflicting data "
1312 "in file (gfid:%s) by %s (%" PRIu64
1313 " bytes @ %s mtime, %s "
1314 "ctime).",
1315 priv->children[fav_child]->name, uuid_utoa(inode->gfid),
1316 policy_str, replies[fav_child].poststat.ia_size, mtime_str,
1317 ctime_str);
1318
1319 sources[fav_child] = 1;
1320 sinks[fav_child] = 0;
1321 healed_sinks[fav_child] = 0;
1322 }
1323 return fav_child;
1324 }
1325
1326 gf_boolean_t
1327 afr_is_file_empty_on_all_children(afr_private_t *priv,
1328 struct afr_reply *replies)
1329 {
1330 int i = 0;
1331
1332 for (i = 0; i < priv->child_count; i++) {
1333 if ((!replies[i].valid) || (replies[i].op_ret != 0) ||
1334 (replies[i].poststat.ia_size != 0))
1335 return _gf_false;
1336 }
1337
1338 return _gf_true;
1339 }
1340
1341 int
1342 afr_mark_source_sinks_if_file_empty(xlator_t *this, unsigned char *sources,
1343 unsigned char *sinks,
1344 unsigned char *healed_sinks,
1345 unsigned char *locked_on,
1346 struct afr_reply *replies,
1347 afr_transaction_type type)
1348 {
1349 int source = -1;
1350 int i = 0;
1351 afr_private_t *priv = this->private;
1352 struct iatt stbuf = {
1353 0,
1354 };
1355
1356 if ((AFR_COUNT(locked_on, priv->child_count) < priv->child_count) ||
1357 (afr_success_count(replies, priv->child_count) < priv->child_count))
1358 return -1;
1359
1360 if (type == AFR_DATA_TRANSACTION) {
1361 if (!afr_is_file_empty_on_all_children(priv, replies))
1362 return -1;
1363 goto mark;
1364 }
1365
1366 /*For AFR_METADATA_TRANSACTION, metadata must be same on all bricks.*/
1367 stbuf = replies[0].poststat;
1368 for (i = 1; i < priv->child_count; i++) {
1369 if ((!IA_EQUAL(stbuf, replies[i].poststat, type)) ||
1370 (!IA_EQUAL(stbuf, replies[i].poststat, uid)) ||
1371 (!IA_EQUAL(stbuf, replies[i].poststat, gid)) ||
1372 (!IA_EQUAL(stbuf, replies[i].poststat, prot)))
1373 return -1;
1374 }
1375 for (i = 1; i < priv->child_count; i++) {
1376 if (!afr_xattrs_are_equal(
1377 replies[0].xdata, replies[i].xdata,
1378 AFR_IS_ARBITER_BRICK(priv, i) ? _gf_true : _gf_false))
1379 return -1;
1380 }
1381
1382 mark:
1383 /* data/metadata is same on all bricks. Pick one of them as source. Rest
1384 * are sinks.*/
1385 for (i = 0; i < priv->child_count; i++) {
1386 if (source == -1) {
1387 source = i;
1388 sources[i] = 1;
1389 sinks[i] = 0;
1390 healed_sinks[i] = 0;
1391 continue;
1392 }
1393 sources[i] = 0;
1394 sinks[i] = 1;
1395 healed_sinks[i] = 1;
1396 }
1397
1398 return source;
1399 }
1400
1401 /* Return a source depending on the type of heal_op, and set sources[source],
1402 * sinks[source] and healed_sinks[source] to 1, 0 and 0 respectively. Do so
1403 * only if the following condition is met:
1404 * ∀i((i ∈ locked_on[] ∧ i=1)==>(sources[i]=0 ∧ sinks[i]=1 ∧ healed_sinks[i]=1))
1405 * i.e. for each locked node, sources[node] is 0; healed_sinks[node] and
1406 * sinks[node] are 1. This should be the case if the file is in split-brain.
1407 */
1408 int
1409 afr_mark_split_brain_source_sinks(
1410 call_frame_t *frame, xlator_t *this, inode_t *inode, unsigned char *sources,
1411 unsigned char *sinks, unsigned char *healed_sinks, unsigned char *locked_on,
1412 struct afr_reply *replies, afr_transaction_type type)
1413 {
1414 afr_local_t *local = NULL;
1415 afr_private_t *priv = NULL;
1416 dict_t *xdata_req = NULL;
1417 int heal_op = -1;
1418 int ret = -1;
1419 int source = -1;
1420
1421 local = frame->local;
1422 priv = this->private;
1423 xdata_req = local->xdata_req;
1424
1425 source = afr_mark_source_sinks_if_file_empty(
1426 this, sources, sinks, healed_sinks, locked_on, replies, type);
1427 if (source >= 0)
1428 return source;
1429
1430 ret = dict_get_int32_sizen(xdata_req, "heal-op", &heal_op);
1431 if (ret)
1432 goto autoheal;
1433
1434 source = afr_mark_split_brain_source_sinks_by_heal_op(
1435 frame, this, sources, sinks, healed_sinks, locked_on, replies, type,
1436 heal_op);
1437 return source;
1438
1439 autoheal:
1440 /* Automatically heal if fav_child_policy is set. */
1441 if (priv->fav_child_policy != AFR_FAV_CHILD_NONE) {
1442 source = afr_mark_split_brain_source_sinks_by_policy(
1443 frame, this, inode, sources, sinks, healed_sinks, locked_on,
1444 replies, type);
1445 if (source != -1) {
1446 ret = dict_set_int32_sizen(xdata_req, "fav-child-policy", 1);
1447 if (ret)
1448 return -1;
1449 }
1450 }
1451
1452 return source;
1453 }
1454
1455 int
1456 _afr_fav_child_reset_sink_xattrs(call_frame_t *frame, xlator_t *this,
1457 inode_t *inode, int source,
1458 unsigned char *healed_sinks,
1459 unsigned char *undid_pending,
1460 afr_transaction_type type,
1461 unsigned char *locked_on,
1462 struct afr_reply *replies)
1463 {
1464 afr_private_t *priv = NULL;
1465 afr_local_t *local = NULL;
1466 int *input_dirty = NULL;
1467 int **input_matrix = NULL;
1468 int *output_dirty = NULL;
1469 int **output_matrix = NULL;
1470 dict_t *xattr = NULL;
1471 dict_t *xdata = NULL;
1472 int i = 0;
1473
1474 priv = this->private;
1475 local = frame->local;
1476
1477 if (!dict_get_sizen(local->xdata_req, "fav-child-policy"))
1478 return 0;
1479
1480 xdata = dict_new();
1481 if (!xdata)
1482 return -1;
1483
1484 input_dirty = alloca0(priv->child_count * sizeof(int));
1485 input_matrix = ALLOC_MATRIX(priv->child_count, int);
1486 output_dirty = alloca0(priv->child_count * sizeof(int));
1487 output_matrix = ALLOC_MATRIX(priv->child_count, int);
1488
1489 afr_selfheal_extract_xattr(this, replies, type, input_dirty, input_matrix);
1490
1491 for (i = 0; i < priv->child_count; i++) {
1492 if (i == source || !healed_sinks[i])
1493 continue;
1494 output_dirty[i] = -input_dirty[i];
1495 output_matrix[i][source] = -input_matrix[i][source];
1496 }
1497
1498 for (i = 0; i < priv->child_count; i++) {
1499 if (!healed_sinks[i] || !locked_on[i])
1500 continue;
1501 xattr = afr_selfheal_output_xattr(this, _gf_false, type, output_dirty,
1502 output_matrix, i, NULL);
1503
1504 afr_selfheal_post_op(frame, this, inode, i, xattr, xdata);
1505
1506 undid_pending[i] = 1;
1507 dict_unref(xattr);
1508 }
1509
1510 if (xdata)
1511 dict_unref(xdata);
1512
1513 return 0;
1514 }
1515
1516 gf_boolean_t
1517 afr_does_witness_exist(xlator_t *this, uint64_t *witness)
1518 {
1519 int i = 0;
1520 afr_private_t *priv = NULL;
1521
1522 priv = this->private;
1523
1524 for (i = 0; i < priv->child_count; i++) {
1525 if (witness[i])
1526 return _gf_true;
1527 }
1528 return _gf_false;
1529 }
1530
1531 unsigned int
1532 afr_get_quorum_count(afr_private_t *priv)
1533 {
1534 if (priv->quorum_count == AFR_QUORUM_AUTO) {
1535 return priv->child_count / 2 + 1;
1536 } else {
1537 return priv->quorum_count;
1538 }
1539 }
1540
1541 static void
1542 afr_selfheal_post_op_failure_accounting(afr_private_t *priv, char *accused,
1543 unsigned char *sources,
1544 unsigned char *locked_on)
1545 {
1546 int i = 0;
1547 unsigned int quorum_count = 0;
1548
1549 if (AFR_COUNT(sources, priv->child_count) != 0)
1550 return;
1551
1552 quorum_count = afr_get_quorum_count(priv);
1553 for (i = 0; i < priv->child_count; i++) {
1554 if ((accused[i] < quorum_count) && locked_on[i]) {
1555 sources[i] = 1;
1556 }
1557 }
1558 return;
1559 }
1560
1561 /*
1562 * This function determines if a self-heal is required for a given inode,
1563 * and if needed, in what direction.
1564 *
1565 * locked_on[] is the array representing servers which have been locked and
1566 * from which xattrs have been fetched for analysis.
1567 *
1568 * The output of the function is by filling the arrays sources[] and sinks[].
1569 *
1570 * sources[i] is set if i'th server is an eligible source for a selfheal.
1571 *
1572 * sinks[i] is set if i'th server needs to be healed.
1573 *
1574 * if sources[0..N] are all set, there is no need for a selfheal.
1575 *
1576 * if sinks[0..N] are all set, the inode is in split brain.
1577 *
1578 */
1579
1580 int
1581 afr_selfheal_find_direction(call_frame_t *frame, xlator_t *this,
1582 struct afr_reply *replies,
1583 afr_transaction_type type, unsigned char *locked_on,
1584 unsigned char *sources, unsigned char *sinks,
1585 uint64_t *witness, unsigned char *pflag)
1586 {
1587 afr_private_t *priv = NULL;
1588 int i = 0;
1589 int j = 0;
1590 int *dirty = NULL; /* Denotes if dirty xattr is set */
1591 int **matrix = NULL; /* Changelog matrix */
1592 char *accused = NULL; /* Accused others without any self-accusal */
1593 char *pending = NULL; /* Have pending operations on others */
1594 char *self_accused = NULL; /* Accused itself */
1595
1596 priv = this->private;
1597
1598 dirty = alloca0(priv->child_count * sizeof(int));
1599 accused = alloca0(priv->child_count);
1600 pending = alloca0(priv->child_count);
1601 self_accused = alloca0(priv->child_count);
1602 matrix = ALLOC_MATRIX(priv->child_count, int);
1603 memset(witness, 0, sizeof(*witness) * priv->child_count);
1604
1605 /* First construct the pending matrix for further analysis */
1606 afr_selfheal_extract_xattr(this, replies, type, dirty, matrix);
1607
1608 if (pflag) {
1609 for (i = 0; i < priv->child_count; i++) {
1610 for (j = 0; j < priv->child_count; j++)
1611 if (matrix[i][j])
1612 *pflag |= PFLAG_PENDING;
1613 if (*pflag)
1614 break;
1615 }
1616 }
1617
1618 if (afr_success_count(replies, priv->child_count) < priv->child_count) {
1619 /* Treat this just like locks not being acquired */
1620 return -ENOTCONN;
1621 }
1622
1623 /* short list all self-accused */
1624 for (i = 0; i < priv->child_count; i++) {
1625 if (matrix[i][i])
1626 self_accused[i] = 1;
1627 }
1628
1629 /* Next short list all accused to exclude them from being sources */
1630 /* Self-accused can't accuse others as they are FOOLs */
1631 for (i = 0; i < priv->child_count; i++) {
1632 for (j = 0; j < priv->child_count; j++) {
1633 if (matrix[i][j]) {
1634 if (!self_accused[i])
1635 accused[j] += 1;
1636 if (i != j)
1637 pending[i] += 1;
1638 }
1639 }
1640 }
1641
1642 /* Short list all non-accused as sources */
1643 for (i = 0; i < priv->child_count; i++) {
1644 if (!accused[i] && locked_on[i])
1645 sources[i] = 1;
1646 else
1647 sources[i] = 0;
1648 }
1649
1650 /* Everyone accused by non-self-accused sources are sinks */
1651 memset(sinks, 0, priv->child_count);
1652 for (i = 0; i < priv->child_count; i++) {
1653 if (!sources[i])
1654 continue;
1655 if (self_accused[i])
1656 continue;
1657 for (j = 0; j < priv->child_count; j++) {
1658 if (matrix[i][j])
1659 sinks[j] = 1;
1660 }
1661 }
1662
1663 /* For breaking ties provide with number of fops they witnessed */
1664
1665 /*
1666 * count the pending fops witnessed from itself to others when it is
1667 * self-accused
1668 */
1669 for (i = 0; i < priv->child_count; i++) {
1670 if (!self_accused[i])
1671 continue;
1672 for (j = 0; j < priv->child_count; j++) {
1673 if (i == j)
1674 continue;
1675 witness[i] += matrix[i][j];
1676 }
1677 }
1678
1679 if (type == AFR_DATA_TRANSACTION || type == AFR_METADATA_TRANSACTION)
1680 afr_selfheal_post_op_failure_accounting(priv, accused, sources,
1681 locked_on);
1682
1683 /* If no sources, all locked nodes are sinks - split brain */
1684 if (AFR_COUNT(sources, priv->child_count) == 0) {
1685 for (i = 0; i < priv->child_count; i++) {
1686 if (locked_on[i])
1687 sinks[i] = 1;
1688 }
1689 if (pflag)
1690 *pflag |= PFLAG_SBRAIN;
1691 }
1692
1693 /* One more class of witness similar to dirty in v2 is where no pending
1694 * exists but we have self-accusing markers. This can happen in afr-v1
1695 * if the brick crashes just after doing xattrop on self but
1696 * before xattrop on the other xattrs on the brick in pre-op. */
1697 if (AFR_COUNT(pending, priv->child_count) == 0) {
1698 for (i = 0; i < priv->child_count; i++) {
1699 if (self_accused[i])
1700 witness[i] += matrix[i][i];
1701 }
1702 } else {
1703 /* In afr-v1 if a file is self-accused and has pending
1704 * operations on others then it is similar to 'dirty' in afr-v2.
1705 * Consider such cases as witness.
1706 */
1707 for (i = 0; i < priv->child_count; i++) {
1708 if (self_accused[i] && pending[i])
1709 witness[i] += matrix[i][i];
1710 }
1711 }
1712
1713 /* count the number of dirty fops witnessed */
1714 for (i = 0; i < priv->child_count; i++)
1715 witness[i] += dirty[i];
1716
1717 return 0;
1718 }
1719
1720 void
1721 afr_log_selfheal(uuid_t gfid, xlator_t *this, int ret, char *type, int source,
1722 unsigned char *sources, unsigned char *healed_sinks)
1723 {
1724 char *status = NULL;
1725 char *sinks_str = NULL;
1726 char *p = NULL;
1727 char *sources_str = NULL;
1728 char *q = NULL;
1729 afr_private_t *priv = NULL;
1730 gf_loglevel_t loglevel = GF_LOG_NONE;
1731 int i = 0;
1732
1733 priv = this->private;
1734 sinks_str = alloca0(priv->child_count * 8);
1735 p = sinks_str;
1736 sources_str = alloca0(priv->child_count * 8);
1737 q = sources_str;
1738 for (i = 0; i < priv->child_count; i++) {
1739 if (healed_sinks[i])
1740 p += sprintf(p, "%d ", i);
1741 if (sources[i]) {
1742 if (source == i) {
1743 q += sprintf(q, "[%d] ", i);
1744 } else {
1745 q += sprintf(q, "%d ", i);
1746 }
1747 }
1748 }
1749
1750 if (ret < 0) {
1751 status = "Failed";
1752 loglevel = GF_LOG_DEBUG;
1753 } else {
1754 status = "Completed";
1755 loglevel = GF_LOG_INFO;
1756 }
1757
1758 gf_msg(this->name, loglevel, 0, AFR_MSG_SELF_HEAL_INFO,
1759 "%s %s selfheal on %s. "
1760 "sources=%s sinks=%s",
1761 status, type, uuid_utoa(gfid), sources_str, sinks_str);
1762 }
1763
1764 int
1765 afr_selfheal_discover_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
1766 int op_ret, int op_errno, inode_t *inode,
1767 struct iatt *buf, dict_t *xdata, struct iatt *parbuf)
1768 {
1769 afr_local_t *local = NULL;
1770 int i = -1;
1771 GF_UNUSED int ret = -1;
1772 int8_t need_heal = 1;
1773
1774 local = frame->local;
1775 i = (long)cookie;
1776
1777 local->replies[i].valid = 1;
1778 local->replies[i].op_ret = op_ret;
1779 local->replies[i].op_errno = op_errno;
1780 if (buf)
1781 local->replies[i].poststat = *buf;
1782 if (parbuf)
1783 local->replies[i].postparent = *parbuf;
1784 if (xdata) {
1785 local->replies[i].xdata = dict_ref(xdata);
1786 ret = dict_get_int8(xdata, "link-count", &need_heal);
1787 }
1788
1789 local->replies[i].need_heal = need_heal;
1790 syncbarrier_wake(&local->barrier);
1791
1792 return 0;
1793 }
1794
1795 inode_t *
1796 afr_selfheal_unlocked_lookup_on(call_frame_t *frame, inode_t *parent,
1797 const char *name, struct afr_reply *replies,
1798 unsigned char *lookup_on, dict_t *xattr)
1799 {
1800 loc_t loc = {
1801 0,
1802 };
1803 dict_t *xattr_req = NULL;
1804 afr_local_t *local = NULL;
1805 afr_private_t *priv = NULL;
1806 inode_t *inode = NULL;
1807
1808 local = frame->local;
1809 priv = frame->this->private;
1810
1811 xattr_req = dict_new();
1812 if (!xattr_req)
1813 return NULL;
1814
1815 if (xattr)
1816 dict_copy(xattr, xattr_req);
1817
1818 if (afr_xattr_req_prepare(frame->this, xattr_req) != 0) {
1819 dict_unref(xattr_req);
1820 return NULL;
1821 }
1822
1823 inode = inode_new(parent->table);
1824 if (!inode) {
1825 dict_unref(xattr_req);
1826 return NULL;
1827 }
1828
1829 loc.parent = inode_ref(parent);
1830 gf_uuid_copy(loc.pargfid, parent->gfid);
1831 loc.name = name;
1832 loc.inode = inode_ref(inode);
1833
1834 AFR_ONLIST(lookup_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
1835 xattr_req);
1836
1837 afr_replies_copy(replies, local->replies, priv->child_count);
1838
1839 loc_wipe(&loc);
1840 dict_unref(xattr_req);
1841
1842 return inode;
1843 }
1844
1845 static int
1846 afr_set_multi_dom_lock_count_request(xlator_t *this, dict_t *dict)
1847 {
1848 int ret = 0;
1849 afr_private_t *priv = NULL;
1850 char *key1 = NULL;
1851 char *key2 = NULL;
1852
1853 priv = this->private;
1854 key1 = alloca0(strlen(GLUSTERFS_INODELK_DOM_PREFIX) + 2 +
1855 strlen(this->name));
1856 key2 = alloca0(strlen(GLUSTERFS_INODELK_DOM_PREFIX) + 2 +
1857 strlen(priv->sh_domain));
1858
1859 ret = dict_set_uint32(dict, GLUSTERFS_MULTIPLE_DOM_LK_CNT_REQUESTS, 1);
1860 if (ret)
1861 return ret;
1862
1863 sprintf(key1, "%s:%s", GLUSTERFS_INODELK_DOM_PREFIX, this->name);
1864 ret = dict_set_uint32(dict, key1, 1);
1865 if (ret)
1866 return ret;
1867
1868 sprintf(key2, "%s:%s", GLUSTERFS_INODELK_DOM_PREFIX, priv->sh_domain);
1869 ret = dict_set_uint32(dict, key2, 1);
1870 if (ret)
1871 return ret;
1872
1873 return 0;
1874 }
1875
1876 int
1877 afr_selfheal_unlocked_discover_on(call_frame_t *frame, inode_t *inode,
1878 uuid_t gfid, struct afr_reply *replies,
1879 unsigned char *discover_on, dict_t *dict)
1880 {
1881 loc_t loc = {
1882 0,
1883 };
1884 dict_t *xattr_req = NULL;
1885 afr_local_t *local = NULL;
1886 afr_private_t *priv = NULL;
1887
1888 local = frame->local;
1889 priv = frame->this->private;
1890
1891 xattr_req = dict_new();
1892 if (!xattr_req)
1893 return -ENOMEM;
1894 if (dict)
1895 dict_copy(dict, xattr_req);
1896
1897 if (afr_xattr_req_prepare(frame->this, xattr_req) != 0) {
1898 dict_unref(xattr_req);
1899 return -ENOMEM;
1900 }
1901
1902 if (afr_set_multi_dom_lock_count_request(frame->this, xattr_req)) {
1903 dict_unref(xattr_req);
1904 return -1;
1905 }
1906
1907 loc.inode = inode_ref(inode);
1908 gf_uuid_copy(loc.gfid, gfid);
1909
1910 AFR_ONLIST(discover_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
1911 xattr_req);
1912
1913 afr_replies_copy(replies, local->replies, priv->child_count);
1914
1915 loc_wipe(&loc);
1916 dict_unref(xattr_req);
1917
1918 return 0;
1919 }
1920
1921 int
1922 afr_selfheal_unlocked_discover(call_frame_t *frame, inode_t *inode, uuid_t gfid,
1923 struct afr_reply *replies)
1924 {
1925 afr_local_t *local = NULL;
1926 dict_t *dict = NULL;
1927
1928 local = frame->local;
1929
1930 if (local->xattr_req)
1931 dict = local->xattr_req;
1932
1933 return afr_selfheal_unlocked_discover_on(frame, inode, gfid, replies,
1934 local->child_up, dict);
1935 }
1936
1937 unsigned int
1938 afr_success_count(struct afr_reply *replies, unsigned int count)
1939 {
1940 int i = 0;
1941 unsigned int success = 0;
1942
1943 for (i = 0; i < count; i++)
1944 if (replies[i].valid && replies[i].op_ret == 0)
1945 success++;
1946 return success;
1947 }
1948
1949 int
1950 afr_selfheal_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
1951 int op_ret, int op_errno, dict_t *xdata)
1952 {
1953 afr_local_t *local = NULL;
1954 int i = 0;
1955
1956 local = frame->local;
1957 i = (long)cookie;
1958
1959 local->replies[i].valid = 1;
1960 local->replies[i].op_ret = op_ret;
1961 local->replies[i].op_errno = op_errno;
1962
1963 syncbarrier_wake(&local->barrier);
1964
1965 return 0;
1966 }
1967
1968 int
1969 afr_locked_fill(call_frame_t *frame, xlator_t *this, unsigned char *locked_on)
1970 {
1971 int i = 0;
1972 afr_private_t *priv = NULL;
1973 afr_local_t *local = NULL;
1974 int count = 0;
1975
1976 local = frame->local;
1977 priv = this->private;
1978
1979 for (i = 0; i < priv->child_count; i++) {
1980 if (local->replies[i].valid && local->replies[i].op_ret == 0) {
1981 locked_on[i] = 1;
1982 count++;
1983 } else {
1984 locked_on[i] = 0;
1985 }
1986 }
1987
1988 return count;
1989 }
1990
1991 int
1992 afr_selfheal_inodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
1993 char *dom, off_t off, size_t size,
1994 unsigned char *locked_on)
1995 {
1996 loc_t loc = {
1997 0,
1998 };
1999 struct gf_flock flock = {
2000 0,
2001 };
2002 afr_local_t *local = NULL;
2003 int i = 0;
2004 afr_private_t *priv = NULL;
2005
2006 priv = this->private;
2007 local = frame->local;
2008
2009 loc.inode = inode_ref(inode);
2010 gf_uuid_copy(loc.gfid, inode->gfid);
2011
2012 flock.l_type = F_WRLCK;
2013 flock.l_start = off;
2014 flock.l_len = size;
2015
2016 AFR_ONALL(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLK, &flock,
2017 NULL);
2018
2019 for (i = 0; i < priv->child_count; i++) {
2020 if (local->replies[i].op_ret == -1 &&
2021 local->replies[i].op_errno == EAGAIN) {
2022 afr_locked_fill(frame, this, locked_on);
2023 afr_selfheal_uninodelk(frame, this, inode, dom, off, size,
2024 locked_on);
2025
2026 AFR_SEQ(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLKW,
2027 &flock, NULL);
2028 break;
2029 }
2030 }
2031
2032 loc_wipe(&loc);
2033
2034 return afr_locked_fill(frame, this, locked_on);
2035 }
2036
2037 static void
2038 afr_get_lock_and_eagain_counts(afr_private_t *priv, struct afr_reply *replies,
2039 int *lock_count, int *eagain_count)
2040 {
2041 int i = 0;
2042
2043 for (i = 0; i < priv->child_count; i++) {
2044 if (!replies[i].valid)
2045 continue;
2046 if (replies[i].op_ret == 0) {
2047 (*lock_count)++;
2048 } else if (replies[i].op_ret == -1 && replies[i].op_errno == EAGAIN) {
2049 (*eagain_count)++;
2050 }
2051 }
2052 }
2053
2054 /*Do blocking locks if number of locks acquired is majority and there were some
2055 * EAGAINs. Useful for odd-way replication*/
2056 int
2057 afr_selfheal_tie_breaker_inodelk(call_frame_t *frame, xlator_t *this,
2058 inode_t *inode, char *dom, off_t off,
2059 size_t size, unsigned char *locked_on)
2060 {
2061 loc_t loc = {
2062 0,
2063 };
2064 struct gf_flock flock = {
2065 0,
2066 };
2067 afr_local_t *local = NULL;
2068 afr_private_t *priv = NULL;
2069 int lock_count = 0;
2070 int eagain_count = 0;
2071
2072 priv = this->private;
2073 local = frame->local;
2074
2075 loc.inode = inode_ref(inode);
2076 gf_uuid_copy(loc.gfid, inode->gfid);
2077
2078 flock.l_type = F_WRLCK;
2079 flock.l_start = off;
2080 flock.l_len = size;
2081
2082 AFR_ONALL(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLK, &flock,
2083 NULL);
2084
2085 afr_get_lock_and_eagain_counts(priv, local->replies, &lock_count,
2086 &eagain_count);
2087
2088 if (lock_count > priv->child_count / 2 && eagain_count) {
2089 afr_locked_fill(frame, this, locked_on);
2090 afr_selfheal_uninodelk(frame, this, inode, dom, off, size, locked_on);
2091
2092 AFR_SEQ(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLKW,
2093 &flock, NULL);
2094 }
2095
2096 loc_wipe(&loc);
2097
2098 return afr_locked_fill(frame, this, locked_on);
2099 }
2100
2101 int
2102 afr_selfheal_uninodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
2103 char *dom, off_t off, size_t size,
2104 const unsigned char *locked_on)
2105 {
2106 loc_t loc = {
2107 0,
2108 };
2109 struct gf_flock flock = {
2110 0,
2111 };
2112
2113 loc.inode = inode_ref(inode);
2114 gf_uuid_copy(loc.gfid, inode->gfid);
2115
2116 flock.l_type = F_UNLCK;
2117 flock.l_start = off;
2118 flock.l_len = size;
2119
2120 AFR_ONLIST(locked_on, frame, afr_selfheal_lock_cbk, inodelk, dom, &loc,
2121 F_SETLK, &flock, NULL);
2122
2123 loc_wipe(&loc);
2124
2125 return 0;
2126 }
2127
2128 int
2129 afr_selfheal_tryentrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
2130 char *dom, const char *name, unsigned char *locked_on)
2131 {
2132 loc_t loc = {
2133 0,
2134 };
2135
2136 loc.inode = inode_ref(inode);
2137 gf_uuid_copy(loc.gfid, inode->gfid);
2138
2139 AFR_ONALL(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
2140 ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
2141
2142 loc_wipe(&loc);
2143
2144 return afr_locked_fill(frame, this, locked_on);
2145 }
2146
2147 int
2148 afr_selfheal_entrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
2149 char *dom, const char *name, unsigned char *locked_on)
2150 {
2151 loc_t loc = {
2152 0,
2153 };
2154 afr_local_t *local = NULL;
2155 int i = 0;
2156 afr_private_t *priv = NULL;
2157
2158 priv = this->private;
2159 local = frame->local;
2160
2161 loc.inode = inode_ref(inode);
2162 gf_uuid_copy(loc.gfid, inode->gfid);
2163
2164 AFR_ONALL(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
2165 ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
2166
2167 for (i = 0; i < priv->child_count; i++) {
2168 if (local->replies[i].op_ret == -1 &&
2169 local->replies[i].op_errno == EAGAIN) {
2170 afr_locked_fill(frame, this, locked_on);
2171 afr_selfheal_unentrylk(frame, this, inode, dom, name, locked_on,
2172 NULL);
2173
2174 AFR_SEQ(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
2175 ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
2176 break;
2177 }
2178 }
2179
2180 loc_wipe(&loc);
2181
2182 return afr_locked_fill(frame, this, locked_on);
2183 }
2184
2185 int
2186 afr_selfheal_tie_breaker_entrylk(call_frame_t *frame, xlator_t *this,
2187 inode_t *inode, char *dom, const char *name,
2188 unsigned char *locked_on)
2189 {
2190 loc_t loc = {
2191 0,
2192 };
2193 afr_local_t *local = NULL;
2194 afr_private_t *priv = NULL;
2195 int lock_count = 0;
2196 int eagain_count = 0;
2197
2198 priv = this->private;
2199 local = frame->local;
2200
2201 loc.inode = inode_ref(inode);
2202 gf_uuid_copy(loc.gfid, inode->gfid);
2203
2204 AFR_ONALL(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
2205 ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
2206
2207 afr_get_lock_and_eagain_counts(priv, local->replies, &lock_count,
2208 &eagain_count);
2209
2210 if (lock_count > priv->child_count / 2 && eagain_count) {
2211 afr_locked_fill(frame, this, locked_on);
2212 afr_selfheal_unentrylk(frame, this, inode, dom, name, locked_on, NULL);
2213
2214 AFR_SEQ(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
2215 ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
2216 }
2217
2218 loc_wipe(&loc);
2219
2220 return afr_locked_fill(frame, this, locked_on);
2221 }
2222
2223 int
2224 afr_selfheal_unentrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
2225 char *dom, const char *name, unsigned char *locked_on,
2226 dict_t *xdata)
2227 {
2228 loc_t loc = {
2229 0,
2230 };
2231
2232 loc.inode = inode_ref(inode);
2233 gf_uuid_copy(loc.gfid, inode->gfid);
2234
2235 AFR_ONLIST(locked_on, frame, afr_selfheal_lock_cbk, entrylk, dom, &loc,
2236 name, ENTRYLK_UNLOCK, ENTRYLK_WRLCK, xdata);
2237
2238 loc_wipe(&loc);
2239
2240 return 0;
2241 }
2242
2243 static gf_boolean_t
2244 afr_is_data_set(xlator_t *this, dict_t *xdata)
2245 {
2246 return afr_is_pending_set(this, xdata, AFR_DATA_TRANSACTION);
2247 }
2248
2249 static gf_boolean_t
2250 afr_is_metadata_set(xlator_t *this, dict_t *xdata)
2251 {
2252 return afr_is_pending_set(this, xdata, AFR_METADATA_TRANSACTION);
2253 }
2254
2255 static gf_boolean_t
2256 afr_is_entry_set(xlator_t *this, dict_t *xdata)
2257 {
2258 return afr_is_pending_set(this, xdata, AFR_ENTRY_TRANSACTION);
2259 }
2260
2261 /*
2262 * This function inspects the looked up replies (in an unlocked manner)
2263 * and decides whether a locked verification and possible healing is
2264 * required or not. It updates the three booleans for each type
2265 * of healing. If the boolean flag gets set to FALSE, then we are sure
2266 * no healing is required. If the boolean flag gets set to TRUE then
2267 * we have to proceed with locked reinspection.
2268 */
2269
2270 int
2271 afr_selfheal_unlocked_inspect(call_frame_t *frame, xlator_t *this, uuid_t gfid,
2272 inode_t **link_inode, gf_boolean_t *data_selfheal,
2273 gf_boolean_t *metadata_selfheal,
2274 gf_boolean_t *entry_selfheal,
2275 struct afr_reply *replies_dst)
2276 {
2277 afr_private_t *priv = NULL;
2278 inode_t *inode = NULL;
2279 int i = 0;
2280 int valid_cnt = 0;
2281 struct iatt first = {
2282 0,
2283 };
2284 int first_idx = 0;
2285 struct afr_reply *replies = NULL;
2286 int ret = -1;
2287
2288 priv = this->private;
2289
2290 inode = afr_inode_find(this, gfid);
2291 if (!inode)
2292 goto out;
2293
2294 replies = alloca0(sizeof(*replies) * priv->child_count);
2295
2296 ret = afr_selfheal_unlocked_discover(frame, inode, gfid, replies);
2297 if (ret)
2298 goto out;
2299
2300 for (i = 0; i < priv->child_count; i++) {
2301 if (!replies[i].valid)
2302 continue;
2303 if (replies[i].op_ret == -1)
2304 continue;
2305
2306 /* The data segment of the changelog can be non-zero to indicate
2307 * the directory needs a full heal. So the check below ensures
2308 * it's not a directory before setting the data_selfheal boolean.
2309 */
2310 if (data_selfheal && !IA_ISDIR(replies[i].poststat.ia_type) &&
2311 afr_is_data_set(this, replies[i].xdata))
2312 *data_selfheal = _gf_true;
2313
2314 if (metadata_selfheal && afr_is_metadata_set(this, replies[i].xdata))
2315 *metadata_selfheal = _gf_true;
2316
2317 if (entry_selfheal && afr_is_entry_set(this, replies[i].xdata))
2318 *entry_selfheal = _gf_true;
2319
2320 valid_cnt++;
2321 if (valid_cnt == 1) {
2322 first = replies[i].poststat;
2323 first_idx = i;
2324 continue;
2325 }
2326
2327 if (!IA_EQUAL(first, replies[i].poststat, type)) {
2328 gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
2329 "TYPE mismatch %d vs %d on %s for gfid:%s",
2330 (int)first.ia_type, (int)replies[i].poststat.ia_type,
2331 priv->children[i]->name,
2332 uuid_utoa(replies[i].poststat.ia_gfid));
2333 gf_event(EVENT_AFR_SPLIT_BRAIN,
2334 "client-pid=%d;"
2335 "subvol=%s;"
2336 "type=file;gfid=%s;"
2337 "ia_type-%d=%s;ia_type-%d=%s",
2338 this->ctx->cmd_args.client_pid, this->name,
2339 uuid_utoa(replies[i].poststat.ia_gfid), first_idx,
2340 gf_inode_type_to_str(first.ia_type), i,
2341 gf_inode_type_to_str(replies[i].poststat.ia_type));
2342 ret = -EIO;
2343 goto out;
2344 }
2345
2346 if (!IA_EQUAL(first, replies[i].poststat, uid)) {
2347 gf_msg_debug(this->name, 0,
2348 "UID mismatch "
2349 "%d vs %d on %s for gfid:%s",
2350 (int)first.ia_uid, (int)replies[i].poststat.ia_uid,
2351 priv->children[i]->name,
2352 uuid_utoa(replies[i].poststat.ia_gfid));
2353
2354 if (metadata_selfheal)
2355 *metadata_selfheal = _gf_true;
2356 }
2357
2358 if (!IA_EQUAL(first, replies[i].poststat, gid)) {
2359 gf_msg_debug(this->name, 0,
2360 "GID mismatch "
2361 "%d vs %d on %s for gfid:%s",
2362 (int)first.ia_uid, (int)replies[i].poststat.ia_uid,
2363 priv->children[i]->name,
2364 uuid_utoa(replies[i].poststat.ia_gfid));
2365
2366 if (metadata_selfheal)
2367 *metadata_selfheal = _gf_true;
2368 }
2369
2370 if (!IA_EQUAL(first, replies[i].poststat, prot)) {
2371 gf_msg_debug(this->name, 0,
2372 "MODE mismatch "
2373 "%d vs %d on %s for gfid:%s",
2374 (int)st_mode_from_ia(first.ia_prot, 0),
2375 (int)st_mode_from_ia(replies[i].poststat.ia_prot, 0),
2376 priv->children[i]->name,
2377 uuid_utoa(replies[i].poststat.ia_gfid));
2378
2379 if (metadata_selfheal)
2380 *metadata_selfheal = _gf_true;
2381 }
2382
2383 if (IA_ISREG(first.ia_type) &&
2384 !IA_EQUAL(first, replies[i].poststat, size)) {
2385 gf_msg_debug(this->name, 0,
2386 "SIZE mismatch "
2387 "%lld vs %lld on %s for gfid:%s",
2388 (long long)first.ia_size,
2389 (long long)replies[i].poststat.ia_size,
2390 priv->children[i]->name,
2391 uuid_utoa(replies[i].poststat.ia_gfid));
2392
2393 if (data_selfheal)
2394 *data_selfheal = _gf_true;
2395 }
2396 }
2397
2398 if (valid_cnt > 0 && link_inode) {
2399 *link_inode = inode_link(inode, NULL, NULL, &first);
2400 if (!*link_inode) {
2401 ret = -EINVAL;
2402 goto out;
2403 }
2404 } else if (valid_cnt < 2) {
2405 ret = afr_check_stale_error(replies, priv);
2406 goto out;
2407 }
2408
2409 ret = 0;
2410 out:
2411 if (replies && replies_dst)
2412 afr_replies_copy(replies_dst, replies, priv->child_count);
2413 if (inode)
2414 inode_unref(inode);
2415 if (replies)
2416 afr_replies_wipe(replies, priv->child_count);
2417
2418 return ret;
2419 }
2420
2421 inode_t *
2422 afr_inode_find(xlator_t *this, uuid_t gfid)
2423 {
2424 inode_table_t *table = NULL;
2425 inode_t *inode = NULL;
2426
2427 table = this->itable;
2428 if (!table)
2429 return NULL;
2430
2431 inode = inode_find(table, gfid);
2432 if (inode)
2433 return inode;
2434
2435 inode = inode_new(table);
2436 if (!inode)
2437 return NULL;
2438
2439 gf_uuid_copy(inode->gfid, gfid);
2440
2441 return inode;
2442 }
2443
2444 call_frame_t *
2445 afr_frame_create(xlator_t *this, int32_t *op_errno)
2446 {
2447 call_frame_t *frame = NULL;
2448 afr_local_t *local = NULL;
2449 pid_t pid = GF_CLIENT_PID_SELF_HEALD;
2450
2451 frame = create_frame(this, this->ctx->pool);
2452 if (!frame) {
2453 if (op_errno)
2454 *op_errno = ENOMEM;
2455 return NULL;
2456 }
2457
2458 local = AFR_FRAME_INIT(frame, (*op_errno));
2459 if (!local) {
2460 STACK_DESTROY(frame->root);
2461 return NULL;
2462 }
2463
2464 syncopctx_setfspid(&pid);
2465
2466 frame->root->pid = pid;
2467
2468 afr_set_lk_owner(frame, this, frame->root);
2469
2470 return frame;
2471 }
2472
2473 int
2474 afr_selfheal_newentry_mark(call_frame_t *frame, xlator_t *this, inode_t *inode,
2475 int source, struct afr_reply *replies,
2476 unsigned char *sources, unsigned char *newentry)
2477 {
2478 int ret = 0;
2479 int i = 0;
2480 afr_private_t *priv = NULL;
2481 dict_t *xattr = NULL;
2482 int **changelog = NULL;
2483
2484 priv = this->private;
2485
2486 gf_uuid_copy(inode->gfid, replies[source].poststat.ia_gfid);
2487
2488 xattr = dict_new();
2489 if (!xattr)
2490 return -ENOMEM;
2491
2492 changelog = afr_mark_pending_changelog(priv, newentry, xattr,
2493 replies[source].poststat.ia_type);
2494
2495 if (!changelog) {
2496 ret = -ENOMEM;
2497 goto out;
2498 }
2499
2500 for (i = 0; i < priv->child_count; i++) {
2501 if (!sources[i])
2502 continue;
2503 ret |= afr_selfheal_post_op(frame, this, inode, i, xattr, NULL);
2504 }
2505 out:
2506 if (changelog)
2507 afr_matrix_cleanup(changelog, priv->child_count);
2508 if (xattr)
2509 dict_unref(xattr);
2510 return ret;
2511 }
2512
2513 int
2514 afr_selfheal_do(call_frame_t *frame, xlator_t *this, uuid_t gfid)
2515 {
2516 int ret = -1;
2517 int entry_ret = 1;
2518 int metadata_ret = 1;
2519 int data_ret = 1;
2520 int or_ret = 0;
2521 inode_t *inode = NULL;
2522 fd_t *fd = NULL;
2523 gf_boolean_t data_selfheal = _gf_false;
2524 gf_boolean_t metadata_selfheal = _gf_false;
2525 gf_boolean_t entry_selfheal = _gf_false;
2526 afr_private_t *priv = NULL;
2527
2528 priv = this->private;
2529
2530 ret = afr_selfheal_unlocked_inspect(frame, this, gfid, &inode,
2531 &data_selfheal, &metadata_selfheal,
2532 &entry_selfheal, NULL);
2533 if (ret)
2534 goto out;
2535
2536 if (!(data_selfheal || metadata_selfheal || entry_selfheal)) {
2537 ret = 2;
2538 goto out;
2539 }
2540
2541 if (inode->ia_type == IA_IFREG) {
2542 ret = afr_selfheal_data_open(this, inode, &fd);
2543 if (!fd) {
2544 ret = -EIO;
2545 goto out;
2546 }
2547 }
2548
2549 gf_msg_debug(
2550 this->name, 0,
2551 "heals needed for %s: [entry-heal=%d, metadata-heal=%d, data-heal=%d]",
2552 uuid_utoa(gfid), entry_selfheal, metadata_selfheal, data_selfheal);
2553
2554 if (data_selfheal && priv->data_self_heal)
2555 data_ret = afr_selfheal_data(frame, this, fd);
2556
2557 if (metadata_selfheal && priv->metadata_self_heal)
2558 metadata_ret = afr_selfheal_metadata(frame, this, inode);
2559
2560 if (entry_selfheal && priv->entry_self_heal)
2561 entry_ret = afr_selfheal_entry(frame, this, inode);
2562
2563 or_ret = (data_ret | metadata_ret | entry_ret);
2564
2565 if (data_ret == -EIO || metadata_ret == -EIO || entry_ret == -EIO)
2566 ret = -EIO;
2567 else if (data_ret == 1 && metadata_ret == 1 && entry_ret == 1)
2568 ret = 1;
2569 else if (or_ret < 0)
2570 ret = or_ret;
2571 else
2572 ret = 0;
2573
2574 out:
2575 if (inode)
2576 inode_unref(inode);
2577 if (fd)
2578 fd_unref(fd);
2579 return ret;
2580 }
2581 /*
2582 * This is the entry point for healing a given GFID. The return values for this
2583 * function are as follows:
2584 * '0' if the self-heal is successful
2585 * '1' if the afr-xattrs are non-zero (due to on-going IO) and no heal is needed
2586 * '2' if the afr-xattrs are all-zero and no heal is needed
2587 * $errno if the heal on the gfid failed.
2588 */
2589
2590 int
2591 afr_selfheal(xlator_t *this, uuid_t gfid)
2592 {
2593 int ret = -1;
2594 call_frame_t *frame = NULL;
2595 afr_local_t *local = NULL;
2596
2597 frame = afr_frame_create(this, NULL);
2598 if (!frame)
2599 return ret;
2600
2601 local = frame->local;
2602 local->xdata_req = dict_new();
2603
2604 ret = afr_selfheal_do(frame, this, gfid);
2605
2606 if (frame)
2607 AFR_STACK_DESTROY(frame);
2608
2609 return ret;
2610 }
2611
2612 afr_local_t *
2613 __afr_dequeue_heals(afr_private_t *priv)
2614 {
2615 afr_local_t *local = NULL;
2616
2617 if (list_empty(&priv->heal_waiting))
2618 goto none;
2619 if ((priv->background_self_heal_count > 0) &&
2620 (priv->healers >= priv->background_self_heal_count))
2621 goto none;
2622
2623 local = list_entry(priv->heal_waiting.next, afr_local_t, healer);
2624 priv->heal_waiters--;
2625 GF_ASSERT(priv->heal_waiters >= 0);
2626 list_del_init(&local->healer);
2627 list_add(&local->healer, &priv->healing);
2628 priv->healers++;
2629 return local;
2630 none:
2631 gf_msg_debug(THIS->name, 0,
2632 "Nothing dequeued. "
2633 "Num healers: %d, Num Waiters: %d",
2634 priv->healers, priv->heal_waiters);
2635 return NULL;
2636 }
2637
2638 static int
2639 afr_refresh_selfheal_wrap(void *opaque)
2640 {
2641 call_frame_t *heal_frame = opaque;
2642 afr_local_t *local = heal_frame->local;
2643 int ret = 0;
2644
2645 ret = afr_selfheal(heal_frame->this, local->refreshinode->gfid);
2646 return ret;
2647 }
2648
2649 static int
2650 afr_refresh_heal_done(int ret, call_frame_t *frame, void *opaque)
2651 {
2652 call_frame_t *heal_frame = opaque;
2653 xlator_t *this = heal_frame->this;
2654 afr_private_t *priv = this->private;
2655 afr_local_t *local = heal_frame->local;
2656
2657 LOCK(&priv->lock);
2658 {
2659 list_del_init(&local->healer);
2660 priv->healers--;
2661 GF_ASSERT(priv->healers >= 0);
2662 local = __afr_dequeue_heals(priv);
2663 }
2664 UNLOCK(&priv->lock);
2665
2666 AFR_STACK_DESTROY(heal_frame);
2667
2668 if (local)
2669 afr_heal_synctask(this, local);
2670 return 0;
2671 }
2672
2673 void
2674 afr_heal_synctask(xlator_t *this, afr_local_t *local)
2675 {
2676 int ret = 0;
2677 call_frame_t *heal_frame = NULL;
2678
2679 heal_frame = local->heal_frame;
2680 ret = synctask_new(this->ctx->env, afr_refresh_selfheal_wrap,
2681 afr_refresh_heal_done, heal_frame, heal_frame);
2682 if (ret < 0)
2683 /* Heal not launched. Will be queued when the next inode
2684 * refresh happens and shd hasn't healed it yet. */
2685 afr_refresh_heal_done(ret, heal_frame, heal_frame);
2686 }
2687
2688 gf_boolean_t
2689 afr_throttled_selfheal(call_frame_t *frame, xlator_t *this)
2690 {
2691 gf_boolean_t can_heal = _gf_true;
2692 afr_private_t *priv = this->private;
2693 afr_local_t *local = frame->local;
2694
2695 LOCK(&priv->lock);
2696 {
2697 if ((priv->background_self_heal_count > 0) &&
2698 (priv->heal_wait_qlen + priv->background_self_heal_count) >
2699 (priv->heal_waiters + priv->healers)) {
2700 list_add_tail(&local->healer, &priv->heal_waiting);
2701 priv->heal_waiters++;
2702 local = __afr_dequeue_heals(priv);
2703 } else {
2704 can_heal = _gf_false;
2705 }
2706 }
2707 UNLOCK(&priv->lock);
2708
2709 if (can_heal) {
2710 if (local)
2711 afr_heal_synctask(this, local);
2712 else
2713 gf_msg_debug(this->name, 0,
2714 "Max number of heals are "
2715 "pending, background self-heal rejected.");
2716 }
2717
2718 return can_heal;
2719 }
2720
2721 int
2722 afr_choose_source_by_policy(afr_private_t *priv, unsigned char *sources,
2723 afr_transaction_type type)
2724 {
2725 int source = -1;
2726 int i = 0;
2727
2728 /* Give preference to local child to save on bandwidth */
2729 for (i = 0; i < priv->child_count; i++) {
2730 if (priv->local[i] && sources[i]) {
2731 if ((type == AFR_DATA_TRANSACTION) && AFR_IS_ARBITER_BRICK(priv, i))
2732 continue;
2733
2734 source = i;
2735 goto out;
2736 }
2737 }
2738
2739 for (i = 0; i < priv->child_count; i++) {
2740 if (sources[i]) {
2741 source = i;
2742 goto out;
2743 }
2744 }
2745 out:
2746 return source;
2747 }
2748
2749 static int
2750 afr_anon_inode_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
2751 int32_t op_ret, int32_t op_errno, inode_t *inode,
2752 struct iatt *buf, struct iatt *preparent,
2753 struct iatt *postparent, dict_t *xdata)
2754 {
2755 afr_local_t *local = frame->local;
2756 int i = (long)cookie;
2757
2758 local->replies[i].valid = 1;
2759 local->replies[i].op_ret = op_ret;
2760 local->replies[i].op_errno = op_errno;
2761 if (op_ret == 0) {
2762 local->op_ret = 0;
2763 local->replies[i].poststat = *buf;
2764 local->replies[i].preparent = *preparent;
2765 local->replies[i].postparent = *postparent;
2766 }
2767 if (xdata) {
2768 local->replies[i].xdata = dict_ref(xdata);
2769 }
2770
2771 syncbarrier_wake(&local->barrier);
2772 return 0;
2773 }
2774
2775 int
2776 afr_anon_inode_create(xlator_t *this, int child, inode_t **linked_inode)
2777 {
2778 call_frame_t *frame = NULL;
2779 afr_local_t *local = NULL;
2780 afr_private_t *priv = this->private;
2781 unsigned char *mkdir_on = alloca0(priv->child_count);
2782 unsigned char *lookup_on = alloca0(priv->child_count);
2783 loc_t loc = {0};
2784 int32_t op_errno = 0;
2785 int32_t child_op_errno = 0;
2786 struct iatt iatt = {0};
2787 dict_t *xdata = NULL;
2788 uuid_t anon_inode_gfid = {0};
2789 int mkdir_count = 0;
2790 int i = 0;
2791
2792 /*Try to mkdir everywhere and return success if the dir exists on 'child'
2793 */
2794
2795 if (!priv->use_anon_inode) {
2796 op_errno = EINVAL;
2797 goto out;
2798 }
2799
2800 frame = afr_frame_create(this, &op_errno);
2801 if (!frame) {
2802 goto out;
2803 }
2804 local = frame->local;
2805 if (!local->child_up[child]) {
2806 /*Other bricks may need mkdir so don't error out yet*/
2807 child_op_errno = ENOTCONN;
2808 }
2809 gf_uuid_parse(priv->anon_gfid_str, anon_inode_gfid);
2810 for (i = 0; i < priv->child_count; i++) {
2811 if (!local->child_up[i])
2812 continue;
2813
2814 if (priv->anon_inode[i]) {
2815 mkdir_on[i] = 0;
2816 } else {
2817 mkdir_on[i] = 1;
2818 mkdir_count++;
2819 }
2820 }
2821
2822 if (mkdir_count == 0) {
2823 *linked_inode = inode_find(this->itable, anon_inode_gfid);
2824 if (*linked_inode) {
2825 op_errno = 0;
2826 goto out;
2827 }
2828 }
2829
2830 loc.parent = inode_ref(this->itable->root);
2831 loc.name = priv->anon_inode_name;
2832 loc.inode = inode_new(this->itable);
2833 if (!loc.inode) {
2834 op_errno = ENOMEM;
2835 goto out;
2836 }
2837
2838 xdata = dict_new();
2839 if (!xdata) {
2840 op_errno = ENOMEM;
2841 goto out;
2842 }
2843
2844 op_errno = -dict_set_gfuuid(xdata, "gfid-req", anon_inode_gfid, _gf_true);
2845 if (op_errno) {
2846 goto out;
2847 }
2848
2849 if (mkdir_count == 0) {
2850 memcpy(lookup_on, local->child_up, priv->child_count);
2851 goto lookup;
2852 }
2853
2854 AFR_ONLIST(mkdir_on, frame, afr_anon_inode_mkdir_cbk, mkdir, &loc, 0755, 0,
2855 xdata);
2856
2857 for (i = 0; i < priv->child_count; i++) {
2858 if (!mkdir_on[i]) {
2859 continue;
2860 }
2861
2862 if (local->replies[i].op_ret == 0) {
2863 priv->anon_inode[i] = 1;
2864 iatt = local->replies[i].poststat;
2865 } else if (local->replies[i].op_ret < 0 &&
2866 local->replies[i].op_errno == EEXIST) {
2867 lookup_on[i] = 1;
2868 } else if (i == child) {
2869 child_op_errno = local->replies[i].op_errno;
2870 }
2871 }
2872
2873 if (AFR_COUNT(lookup_on, priv->child_count) == 0) {
2874 goto link;
2875 }
2876
2877 lookup:
2878 AFR_ONLIST(lookup_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
2879 xdata);
2880 for (i = 0; i < priv->child_count; i++) {
2881 if (!lookup_on[i]) {
2882 continue;
2883 }
2884
2885 if (local->replies[i].op_ret == 0) {
2886 if (gf_uuid_compare(anon_inode_gfid,
2887 local->replies[i].poststat.ia_gfid) == 0) {
2888 priv->anon_inode[i] = 1;
2889 iatt = local->replies[i].poststat;
2890 } else {
2891 if (i == child)
2892 child_op_errno = EINVAL;
2893 gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_DATA,
2894 "%s has gfid: %s", priv->anon_inode_name,
2895 uuid_utoa(local->replies[i].poststat.ia_gfid));
2896 }
2897 } else if (i == child) {
2898 child_op_errno = local->replies[i].op_errno;
2899 }
2900 }
2901 link:
2902 if (!gf_uuid_is_null(iatt.ia_gfid)) {
2903 *linked_inode = inode_link(loc.inode, loc.parent, loc.name, &iatt);
2904 if (*linked_inode) {
2905 op_errno = 0;
2906 inode_lookup(*linked_inode);
2907 } else {
2908 op_errno = ENOMEM;
2909 }
2910 goto out;
2911 }
2912
2913 out:
2914 if (xdata)
2915 dict_unref(xdata);
2916 loc_wipe(&loc);
2917 /*child_op_errno takes precedence*/
2918 if (child_op_errno == 0) {
2919 child_op_errno = op_errno;
2920 }
2921
2922 if (child_op_errno && *linked_inode) {
2923 inode_unref(*linked_inode);
2924 *linked_inode = NULL;
2925 }
2926 if (frame)
2927 AFR_STACK_DESTROY(frame);
2928 return -child_op_errno;
2929 }