Debian Codegraph @

xlators/cluster/afr/src/afr-self-heald.c

raw
1 /*
2 Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
3 This file is part of GlusterFS.
4
5 This file is licensed to you under your choice of the GNU Lesser
6 General Public License, version 3 or any later version (LGPLv3 or
7 later), or the GNU General Public License, version 2 (GPLv2), in all
8 cases as published by the Free Software Foundation.
9 */
10
11 #include "afr.h"
12 #include "afr-self-heal.h"
13 #include "afr-self-heald.h"
14 #include "protocol-common.h"
15 #include <glusterfs/syncop-utils.h>
16 #include "afr-messages.h"
17
18 #define AFR_EH_SPLIT_BRAIN_LIMIT 1024
19 #define AFR_STATISTICS_HISTORY_SIZE 50
20
21 #define ASSERT_LOCAL(this, healer) \
22 if (!afr_shd_is_subvol_local(this, healer->subvol)) { \
23 healer->local = _gf_false; \
24 if (safe_break(healer)) { \
25 break; \
26 } else { \
27 continue; \
28 } \
29 } else { \
30 healer->local = _gf_true; \
31 }
32
33 #define NTH_INDEX_HEALER(this, n) \
34 &((((afr_private_t *)this->private))->shd.index_healers[n])
35 #define NTH_FULL_HEALER(this, n) \
36 &((((afr_private_t *)this->private))->shd.full_healers[n])
37
38 static char *
39 afr_subvol_name(xlator_t *this, int subvol)
40 {
41 afr_private_t *priv = NULL;
42
43 priv = this->private;
44 if (subvol < 0 || subvol > priv->child_count)
45 return NULL;
46
47 return priv->children[subvol]->name;
48 }
49
50 static void
51 afr_destroy_crawl_event_data(void *data)
52 {
53 return;
54 }
55
56 static void
57 afr_destroy_shd_event_data(void *data)
58 {
59 shd_event_t *shd_event = data;
60
61 if (!shd_event)
62 return;
63 GF_FREE(shd_event->path);
64
65 return;
66 }
67
68 static gf_boolean_t
69 afr_shd_is_subvol_local(xlator_t *this, int subvol)
70 {
71 afr_private_t *priv = NULL;
72 gf_boolean_t is_local = _gf_false;
73 loc_t loc = {
74 0,
75 };
76
77 loc.inode = this->itable->root;
78 gf_uuid_copy(loc.gfid, loc.inode->gfid);
79 priv = this->private;
80 syncop_is_subvol_local(priv->children[subvol], &loc, &is_local);
81 return is_local;
82 }
83
84 static int
85 __afr_shd_healer_wait(struct subvol_healer *healer)
86 {
87 afr_private_t *priv = NULL;
88 struct timespec wait_till = {
89 0,
90 };
91 int ret = 0;
92
93 priv = healer->this->private;
94
95 disabled_loop:
96 wait_till.tv_sec = gf_time() + priv->shd.timeout;
97
98 while (!healer->rerun) {
99 ret = pthread_cond_timedwait(&healer->cond, &healer->mutex, &wait_till);
100 if (ret == ETIMEDOUT)
101 break;
102 }
103
104 ret = healer->rerun;
105 healer->rerun = 0;
106
107 if (!priv->shd.enabled)
108 goto disabled_loop;
109
110 return ret;
111 }
112
113 static int
114 afr_shd_healer_wait(struct subvol_healer *healer)
115 {
116 int ret = 0;
117
118 pthread_mutex_lock(&healer->mutex);
119 {
120 ret = __afr_shd_healer_wait(healer);
121 }
122 pthread_mutex_unlock(&healer->mutex);
123
124 return ret;
125 }
126
127 static gf_boolean_t
128 safe_break(struct subvol_healer *healer)
129 {
130 gf_boolean_t ret = _gf_false;
131
132 pthread_mutex_lock(&healer->mutex);
133 {
134 if (healer->rerun)
135 goto unlock;
136
137 healer->running = _gf_false;
138 ret = _gf_true;
139 }
140 unlock:
141 pthread_mutex_unlock(&healer->mutex);
142
143 return ret;
144 }
145
146 static inode_t *
147 afr_shd_inode_find(xlator_t *this, xlator_t *subvol, uuid_t gfid)
148 {
149 int ret = 0;
150 uint64_t val = IA_INVAL;
151 dict_t *xdata = NULL;
152 dict_t *rsp_dict = NULL;
153 inode_t *inode = NULL;
154
155 xdata = dict_new();
156 if (!xdata)
157 goto out;
158
159 ret = dict_set_int8(xdata, GF_INDEX_IA_TYPE_GET_REQ, 1);
160 if (ret)
161 goto out;
162
163 ret = syncop_inode_find(this, subvol, gfid, &inode, xdata, &rsp_dict);
164 if (ret < 0)
165 goto out;
166
167 if (rsp_dict) {
168 ret = dict_get_uint64(rsp_dict, GF_INDEX_IA_TYPE_GET_RSP, &val);
169 if (ret)
170 goto out;
171 }
172 ret = inode_ctx_set2(inode, subvol, 0, &val);
173 out:
174 if (ret && inode) {
175 inode_unref(inode);
176 inode = NULL;
177 }
178 if (xdata)
179 dict_unref(xdata);
180 if (rsp_dict)
181 dict_unref(rsp_dict);
182 return inode;
183 }
184
185 static inode_t *
186 afr_shd_index_inode(xlator_t *this, xlator_t *subvol, char *vgfid)
187 {
188 loc_t rootloc = {
189 0,
190 };
191 inode_t *inode = NULL;
192 int ret = 0;
193 dict_t *xattr = NULL;
194 void *index_gfid = NULL;
195
196 rootloc.inode = inode_ref(this->itable->root);
197 gf_uuid_copy(rootloc.gfid, rootloc.inode->gfid);
198
199 ret = syncop_getxattr(subvol, &rootloc, &xattr, vgfid, NULL, NULL);
200 if (ret || !xattr) {
201 errno = -ret;
202 goto out;
203 }
204
205 ret = dict_get_ptr(xattr, vgfid, &index_gfid);
206 if (ret)
207 goto out;
208
209 gf_msg_debug(this->name, 0, "%s dir gfid for %s: %s", vgfid, subvol->name,
210 uuid_utoa(index_gfid));
211
212 inode = afr_shd_inode_find(this, subvol, index_gfid);
213
214 out:
215 loc_wipe(&rootloc);
216
217 if (xattr)
218 dict_unref(xattr);
219
220 return inode;
221 }
222
223 int
224 afr_shd_entry_purge(xlator_t *subvol, inode_t *inode, char *name,
225 ia_type_t type)
226 {
227 int ret = 0;
228 loc_t loc = {
229 0,
230 };
231
232 loc.parent = inode_ref(inode);
233 loc.name = name;
234
235 if (IA_ISDIR(type))
236 ret = syncop_rmdir(subvol, &loc, 1, NULL, NULL);
237 else
238 ret = syncop_unlink(subvol, &loc, NULL, NULL);
239
240 loc_wipe(&loc);
241 return ret;
242 }
243
244 static void
245 afr_shd_zero_xattrop(xlator_t *this, uuid_t gfid)
246 {
247 call_frame_t *frame = NULL;
248 inode_t *inode = NULL;
249 afr_private_t *priv = NULL;
250 dict_t *xattr = NULL;
251 int ret = 0;
252 int i = 0;
253 int raw[AFR_NUM_CHANGE_LOGS] = {0};
254
255 priv = this->private;
256 frame = afr_frame_create(this, NULL);
257 if (!frame)
258 goto out;
259 inode = afr_inode_find(this, gfid);
260 if (!inode)
261 goto out;
262 xattr = dict_new();
263 if (!xattr)
264 goto out;
265 ret = dict_set_static_bin(xattr, AFR_DIRTY, raw,
266 sizeof(int) * AFR_NUM_CHANGE_LOGS);
267 if (ret)
268 goto out;
269 for (i = 0; i < priv->child_count; i++) {
270 ret = dict_set_static_bin(xattr, priv->pending_key[i], raw,
271 sizeof(int) * AFR_NUM_CHANGE_LOGS);
272 if (ret)
273 goto out;
274 }
275
276 /*Send xattrop to all bricks. Doing a lookup to see if bricks are up or
277 * has valid repies for this gfid seems a bit of an overkill.*/
278 for (i = 0; i < priv->child_count; i++)
279 afr_selfheal_post_op(frame, this, inode, i, xattr, NULL);
280
281 out:
282 if (frame)
283 AFR_STACK_DESTROY(frame);
284 if (inode)
285 inode_unref(inode);
286 if (xattr)
287 dict_unref(xattr);
288 return;
289 }
290
291 static int
292 afr_shd_selfheal_name(struct subvol_healer *healer, int child, uuid_t parent,
293 const char *bname)
294 {
295 int ret = -1;
296
297 ret = afr_selfheal_name(THIS, parent, bname, NULL, NULL, NULL);
298
299 return ret;
300 }
301
302 static int
303 afr_shd_selfheal(struct subvol_healer *healer, int child, uuid_t gfid)
304 {
305 int ret = 0;
306 eh_t *eh = NULL;
307 afr_private_t *priv = NULL;
308 afr_self_heald_t *shd = NULL;
309 shd_event_t *shd_event = NULL;
310 char *path = NULL;
311 xlator_t *subvol = NULL;
312 xlator_t *this = NULL;
313 crawl_event_t *crawl_event = NULL;
314
315 this = healer->this;
316 priv = this->private;
317 shd = &priv->shd;
318 crawl_event = &healer->crawl_event;
319
320 subvol = priv->children[child];
321
322 // If this fails with ENOENT/ESTALE index is stale
323 ret = syncop_gfid_to_path(this->itable, subvol, gfid, &path);
324 if (ret < 0)
325 return ret;
326
327 ret = afr_selfheal(this, gfid);
328
329 LOCK(&priv->lock);
330 {
331 if (ret == -EIO) {
332 eh = shd->split_brain;
333 crawl_event->split_brain_count++;
334 } else if (ret < 0) {
335 crawl_event->heal_failed_count++;
336 } else if (ret == 0) {
337 crawl_event->healed_count++;
338 }
339 }
340 UNLOCK(&priv->lock);
341
342 if (eh) {
343 shd_event = GF_CALLOC(1, sizeof(*shd_event), gf_afr_mt_shd_event_t);
344 if (!shd_event)
345 goto out;
346
347 shd_event->child = child;
348 shd_event->path = path;
349
350 if (eh_save_history(eh, shd_event) < 0)
351 goto out;
352
353 shd_event = NULL;
354 path = NULL;
355 }
356 out:
357 GF_FREE(shd_event);
358 GF_FREE(path);
359 return ret;
360 }
361
362 static void
363 afr_shd_sweep_prepare(struct subvol_healer *healer)
364 {
365 crawl_event_t *event = NULL;
366
367 event = &healer->crawl_event;
368
369 event->healed_count = 0;
370 event->split_brain_count = 0;
371 event->heal_failed_count = 0;
372
373 event->start_time = gf_time();
374 event->end_time = 0;
375 _mask_cancellation();
376 }
377
378 static void
379 afr_shd_sweep_done(struct subvol_healer *healer)
380 {
381 crawl_event_t *event = NULL;
382 crawl_event_t *history = NULL;
383 afr_self_heald_t *shd = NULL;
384
385 event = &healer->crawl_event;
386 shd = &(((afr_private_t *)healer->this->private)->shd);
387
388 event->end_time = gf_time();
389 history = gf_memdup(event, sizeof(*event));
390 event->start_time = 0;
391
392 if (!history)
393 return;
394
395 if (eh_save_history(shd->statistics[healer->subvol], history) < 0)
396 GF_FREE(history);
397 _unmask_cancellation();
398 }
399
400 int
401 afr_shd_index_heal(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
402 void *data)
403 {
404 struct subvol_healer *healer = data;
405 afr_private_t *priv = NULL;
406 uuid_t gfid = {0};
407 int ret = 0;
408 uint64_t val = IA_INVAL;
409
410 priv = healer->this->private;
411 if (!priv->shd.enabled)
412 return -EBUSY;
413
414 gf_msg_debug(healer->this->name, 0, "got entry: %s from %s", entry->d_name,
415 priv->children[healer->subvol]->name);
416
417 ret = gf_uuid_parse(entry->d_name, gfid);
418 if (ret)
419 return 0;
420
421 inode_ctx_get2(parent->inode, subvol, NULL, &val);
422
423 ret = afr_shd_selfheal(healer, healer->subvol, gfid);
424
425 if (ret == -ENOENT || ret == -ESTALE)
426 afr_shd_entry_purge(subvol, parent->inode, entry->d_name, val);
427
428 if (ret == 2)
429 /* If bricks crashed in pre-op after creating indices/xattrop
430 * link but before setting afr changelogs, we end up with stale
431 * xattrop links but zero changelogs. Remove such entries by
432 * sending a post-op with zero changelogs.
433 */
434 afr_shd_zero_xattrop(healer->this, gfid);
435
436 return 0;
437 }
438
439 static int
440 afr_shd_index_sweep(struct subvol_healer *healer, char *vgfid)
441 {
442 loc_t loc = {0};
443 afr_private_t *priv = NULL;
444 int ret = 0;
445 xlator_t *subvol = NULL;
446 dict_t *xdata = NULL;
447 call_frame_t *frame = NULL;
448
449 priv = healer->this->private;
450 subvol = priv->children[healer->subvol];
451
452 frame = afr_frame_create(healer->this, &ret);
453 if (!frame) {
454 ret = -ret;
455 goto out;
456 }
457
458 loc.inode = afr_shd_index_inode(healer->this, subvol, vgfid);
459 if (!loc.inode) {
460 gf_msg(healer->this->name, GF_LOG_WARNING, 0,
461 AFR_MSG_INDEX_DIR_GET_FAILED, "unable to get index-dir on %s",
462 subvol->name);
463 ret = -errno;
464 goto out;
465 }
466
467 xdata = dict_new();
468 if (!xdata || dict_set_int32_sizen(xdata, "get-gfid-type", 1)) {
469 ret = -ENOMEM;
470 goto out;
471 }
472
473 ret = syncop_mt_dir_scan(frame, subvol, &loc, GF_CLIENT_PID_SELF_HEALD,
474 healer, afr_shd_index_heal, xdata,
475 priv->shd.max_threads, priv->shd.wait_qlength);
476
477 if (ret == 0)
478 ret = healer->crawl_event.healed_count;
479
480 out:
481 loc_wipe(&loc);
482
483 if (xdata)
484 dict_unref(xdata);
485 if (frame)
486 AFR_STACK_DESTROY(frame);
487 return ret;
488 }
489
490 static int
491 afr_shd_index_sweep_all(struct subvol_healer *healer)
492 {
493 int ret = 0;
494 int count = 0;
495
496 ret = afr_shd_index_sweep(healer, GF_XATTROP_INDEX_GFID);
497 if (ret < 0)
498 goto out;
499 count = ret;
500
501 ret = afr_shd_index_sweep(healer, GF_XATTROP_DIRTY_GFID);
502 if (ret < 0)
503 goto out;
504 count += ret;
505
506 ret = afr_shd_index_sweep(healer, GF_XATTROP_ENTRY_CHANGES_GFID);
507 if (ret < 0)
508 goto out;
509 count += ret;
510 out:
511 if (ret < 0)
512 return ret;
513 else
514 return count;
515 }
516
517 static int
518 afr_shd_full_heal(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
519 void *data)
520 {
521 struct subvol_healer *healer = data;
522 xlator_t *this = healer->this;
523 afr_private_t *priv = NULL;
524
525 priv = this->private;
526
527 if (this->cleanup_starting) {
528 return -ENOTCONN;
529 }
530
531 if (!priv->shd.enabled)
532 return -EBUSY;
533
534 afr_shd_selfheal_name(healer, healer->subvol, parent->inode->gfid,
535 entry->d_name);
536
537 afr_shd_selfheal(healer, healer->subvol, entry->d_stat.ia_gfid);
538
539 return 0;
540 }
541
542 static int
543 afr_shd_full_sweep(struct subvol_healer *healer, inode_t *inode)
544 {
545 afr_private_t *priv = NULL;
546 loc_t loc = {0};
547
548 priv = healer->this->private;
549 loc.inode = inode;
550 return syncop_ftw(priv->children[healer->subvol], &loc,
551 GF_CLIENT_PID_SELF_HEALD, healer, afr_shd_full_heal);
552 }
553
554 static int
555 afr_shd_fill_ta_loc(xlator_t *this, loc_t *loc)
556 {
557 afr_private_t *priv = NULL;
558 struct iatt stbuf = {
559 0,
560 };
561 int ret = -1;
562
563 priv = this->private;
564 loc->parent = inode_ref(this->itable->root);
565 gf_uuid_copy(loc->pargfid, loc->parent->gfid);
566 loc->name = priv->pending_key[THIN_ARBITER_BRICK_INDEX];
567 loc->inode = inode_new(loc->parent->table);
568 GF_CHECK_ALLOC(loc->inode, ret, out);
569
570 if (!gf_uuid_is_null(priv->ta_gfid))
571 goto assign_gfid;
572
573 ret = syncop_lookup(priv->children[THIN_ARBITER_BRICK_INDEX], loc, &stbuf,
574 0, 0, 0);
575 if (ret) {
576 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
577 "Failed lookup on file %s.", loc->name);
578 goto out;
579 }
580
581 gf_uuid_copy(priv->ta_gfid, stbuf.ia_gfid);
582
583 assign_gfid:
584 gf_uuid_copy(loc->gfid, priv->ta_gfid);
585 ret = 0;
586
587 out:
588 if (ret)
589 loc_wipe(loc);
590
591 return ret;
592 }
593
594 static int
595 _afr_shd_ta_get_xattrs(xlator_t *this, loc_t *loc, dict_t **xdata)
596 {
597 afr_private_t *priv = NULL;
598 dict_t *xattr = NULL;
599 int raw[AFR_NUM_CHANGE_LOGS] = {
600 0,
601 };
602 int ret = -1;
603 int i = 0;
604
605 priv = this->private;
606
607 xattr = dict_new();
608 if (!xattr) {
609 gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_DICT_GET_FAILED,
610 "Failed to create dict.");
611 goto out;
612 }
613 for (i = 0; i < priv->child_count; i++) {
614 ret = dict_set_static_bin(xattr, priv->pending_key[i], &raw,
615 AFR_NUM_CHANGE_LOGS * sizeof(int));
616 if (ret)
617 goto out;
618 }
619
620 ret = syncop_xattrop(priv->children[THIN_ARBITER_BRICK_INDEX], loc,
621 GF_XATTROP_ADD_ARRAY, xattr, NULL, xdata, NULL);
622 if (ret || !(*xdata)) {
623 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
624 "Xattrop failed on %s.", loc->name);
625 }
626
627 out:
628 if (xattr)
629 dict_unref(xattr);
630
631 return ret;
632 }
633
634 static void
635 afr_shd_ta_get_xattrs(xlator_t *this, loc_t *loc, struct subvol_healer *healer,
636 dict_t **xdata)
637 {
638 int ret = 0;
639
640 loc_wipe(loc);
641 if (afr_shd_fill_ta_loc(this, loc)) {
642 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
643 "Failed to populate thin-arbiter loc for: %s.", loc->name);
644 ret = -1;
645 goto out;
646 }
647
648 ret = afr_ta_post_op_lock(this, loc);
649 if (ret)
650 goto out;
651
652 ret = _afr_shd_ta_get_xattrs(this, loc, xdata);
653 if (ret) {
654 if (*xdata) {
655 dict_unref(*xdata);
656 *xdata = NULL;
657 }
658 }
659
660 afr_ta_post_op_unlock(this, loc);
661
662 out:
663 if (ret)
664 healer->rerun = 1;
665 }
666
667 static int
668 afr_shd_ta_unset_xattrs(xlator_t *this, loc_t *loc, dict_t **xdata, int healer)
669 {
670 afr_private_t *priv = NULL;
671 dict_t *xattr = NULL;
672 gf_boolean_t need_xattrop = _gf_false;
673 void *pending_raw = NULL;
674 int *raw = NULL;
675 int i = 0;
676 int j = 0;
677 int val = 0;
678 int ret = -1;
679
680 priv = this->private;
681
682 xattr = dict_new();
683 if (!xattr) {
684 goto out;
685 }
686
687 for (i = 0; i < priv->child_count; i++) {
688 raw = GF_CALLOC(AFR_NUM_CHANGE_LOGS, sizeof(int), gf_afr_mt_int32_t);
689 if (!raw) {
690 goto out;
691 }
692
693 ret = dict_get_ptr(*xdata, priv->pending_key[i], &pending_raw);
694 if (ret) {
695 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
696 "Error getting value "
697 "of pending key %s",
698 priv->pending_key[i]);
699 GF_FREE(raw);
700 goto out;
701 }
702
703 for (j = 0; j < AFR_NUM_CHANGE_LOGS; j++) {
704 val = be32toh(*((int *)pending_raw + j));
705 if (val) {
706 if (i == healer) {
707 gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_THIN_ARB,
708 "I am "
709 "not the good shd. Skipping. "
710 "SHD = %d.",
711 healer);
712 ret = 0;
713 GF_FREE(raw);
714 goto out;
715 }
716 need_xattrop = _gf_true;
717 raw[j] = htobe32(-val);
718 }
719 }
720
721 ret = dict_set_bin(xattr, priv->pending_key[i], raw,
722 AFR_NUM_CHANGE_LOGS * sizeof(int));
723 if (ret) {
724 GF_FREE(raw);
725 goto out;
726 }
727
728 if (need_xattrop)
729 break;
730 }
731
732 if (!need_xattrop) {
733 ret = 0;
734 goto out;
735 }
736
737 ret = syncop_xattrop(priv->children[THIN_ARBITER_BRICK_INDEX], loc,
738 GF_XATTROP_ADD_ARRAY, xattr, NULL, NULL, NULL);
739 if (ret)
740 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
741 "Xattrop failed.");
742
743 out:
744 if (xattr)
745 dict_unref(xattr);
746
747 return ret;
748 }
749
750 static void
751 afr_shd_ta_check_and_unset_xattrs(xlator_t *this, loc_t *loc,
752 struct subvol_healer *healer,
753 dict_t *pre_crawl_xdata)
754 {
755 int ret_lock = 0;
756 int ret = 0;
757 dict_t *post_crawl_xdata = NULL;
758
759 ret_lock = afr_ta_post_op_lock(this, loc);
760 if (ret_lock)
761 goto unref;
762
763 ret = _afr_shd_ta_get_xattrs(this, loc, &post_crawl_xdata);
764 if (ret)
765 goto unref;
766
767 if (!are_dicts_equal(pre_crawl_xdata, post_crawl_xdata, NULL, NULL, NULL)) {
768 ret = -1;
769 goto unref;
770 }
771
772 ret = afr_shd_ta_unset_xattrs(this, loc, &post_crawl_xdata, healer->subvol);
773
774 unref:
775 if (post_crawl_xdata) {
776 dict_unref(post_crawl_xdata);
777 post_crawl_xdata = NULL;
778 }
779
780 if (ret || ret_lock)
781 healer->rerun = 1;
782
783 if (!ret_lock)
784 afr_ta_post_op_unlock(this, loc);
785 }
786
787 static gf_boolean_t
788 afr_bricks_available_for_heal(afr_private_t *priv)
789 {
790 int up_children = 0;
791
792 up_children = __afr_get_up_children_count(priv);
793 if (up_children < 2) {
794 return _gf_false;
795 }
796 return _gf_true;
797 }
798
799 static gf_boolean_t
800 afr_shd_ta_needs_heal(xlator_t *this, struct subvol_healer *healer)
801 {
802 dict_t *xdata = NULL;
803 afr_private_t *priv = NULL;
804 loc_t loc = {
805 0,
806 };
807 int ret = -1;
808 int i = 0;
809 gf_boolean_t need_heal = _gf_false;
810
811 priv = this->private;
812
813 ret = afr_shd_fill_ta_loc(this, &loc);
814 if (ret) {
815 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
816 "Failed to populate thin-arbiter loc for: %s.", loc.name);
817 healer->rerun = 1;
818 goto out;
819 }
820
821 if (_afr_shd_ta_get_xattrs(this, &loc, &xdata)) {
822 healer->rerun = 1;
823 goto out;
824 }
825
826 for (i = 0; i < priv->child_count; i++) {
827 if (afr_ta_dict_contains_pending_xattr(xdata, priv, i)) {
828 need_heal = _gf_true;
829 break;
830 }
831 }
832
833 out:
834 if (xdata)
835 dict_unref(xdata);
836 loc_wipe(&loc);
837
838 return need_heal;
839 }
840
841 static int
842 afr_shd_anon_inode_cleaner(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
843 void *data)
844 {
845 struct subvol_healer *healer = data;
846 afr_private_t *priv = healer->this->private;
847 call_frame_t *frame = NULL;
848 afr_local_t *local = NULL;
849 int ret = 0;
850 loc_t loc = {0};
851 int count = 0;
852 int i = 0;
853 int op_errno = 0;
854 struct iatt *iatt = NULL;
855 gf_boolean_t multiple_links = _gf_false;
856 unsigned char *gfid_present = alloca0(priv->child_count);
857 unsigned char *entry_present = alloca0(priv->child_count);
858 char *type = "file";
859
860 frame = afr_frame_create(healer->this, &ret);
861 if (!frame) {
862 ret = -ret;
863 goto out;
864 }
865 local = frame->local;
866 if (AFR_COUNT(local->child_up, priv->child_count) != priv->child_count) {
867 gf_msg_debug(healer->this->name, 0,
868 "Not all bricks are up. Skipping "
869 "cleanup of %s on %s",
870 entry->d_name, subvol->name);
871 ret = 0;
872 goto out;
873 }
874
875 loc.inode = inode_new(parent->inode->table);
876 if (!loc.inode) {
877 ret = -ENOMEM;
878 goto out;
879 }
880 ret = gf_uuid_parse(entry->d_name, loc.gfid);
881 if (ret) {
882 ret = 0;
883 goto out;
884 }
885 AFR_ONLIST(local->child_up, frame, afr_selfheal_discover_cbk, lookup, &loc,
886 NULL);
887 for (i = 0; i < priv->child_count; i++) {
888 if (local->replies[i].op_ret == 0) {
889 count++;
890 gfid_present[i] = 1;
891 iatt = &local->replies[i].poststat;
892 if (iatt->ia_type == IA_IFDIR) {
893 type = "dir";
894 }
895
896 if (i == healer->subvol) {
897 if (local->replies[i].poststat.ia_nlink > 1) {
898 multiple_links = _gf_true;
899 }
900 }
901 } else if (local->replies[i].op_errno != ENOENT &&
902 local->replies[i].op_errno != ESTALE) {
903 /*We don't have complete view. Skip the entry*/
904 gf_msg_debug(healer->this->name, local->replies[i].op_errno,
905 "Skipping cleanup of %s on %s", entry->d_name,
906 subvol->name);
907 ret = 0;
908 goto out;
909 }
910 }
911
912 /*Inode is deleted from subvol*/
913 if (count == 1 || (iatt && iatt->ia_type != IA_IFDIR && multiple_links)) {
914 gf_msg(healer->this->name, GF_LOG_WARNING, 0,
915 AFR_MSG_EXPUNGING_FILE_OR_DIR, "expunging %s %s/%s on %s", type,
916 priv->anon_inode_name, entry->d_name, subvol->name);
917 ret = afr_shd_entry_purge(subvol, parent->inode, entry->d_name,
918 iatt->ia_type);
919 if (ret == -ENOENT || ret == -ESTALE)
920 ret = 0;
921 } else if (count > 1) {
922 loc_wipe(&loc);
923 loc.parent = inode_ref(parent->inode);
924 loc.name = entry->d_name;
925 loc.inode = inode_new(parent->inode->table);
926 if (!loc.inode) {
927 ret = -ENOMEM;
928 goto out;
929 }
930 AFR_ONLIST(local->child_up, frame, afr_selfheal_discover_cbk, lookup,
931 &loc, NULL);
932 count = 0;
933 for (i = 0; i < priv->child_count; i++) {
934 if (local->replies[i].op_ret == 0) {
935 count++;
936 entry_present[i] = 1;
937 iatt = &local->replies[i].poststat;
938 } else if (local->replies[i].op_errno != ENOENT &&
939 local->replies[i].op_errno != ESTALE) {
940 /*We don't have complete view. Skip the entry*/
941 gf_msg_debug(healer->this->name, local->replies[i].op_errno,
942 "Skipping cleanup of %s on %s", entry->d_name,
943 subvol->name);
944 ret = 0;
945 goto out;
946 }
947 }
948 for (i = 0; i < priv->child_count; i++) {
949 if (gfid_present[i] && !entry_present[i]) {
950 /*Entry is not anonymous on at least one subvol*/
951 gf_msg_debug(healer->this->name, 0,
952 "Valid entry present on %s "
953 "Skipping cleanup of %s on %s",
954 priv->children[i]->name, entry->d_name,
955 subvol->name);
956 ret = 0;
957 goto out;
958 }
959 }
960
961 gf_msg(healer->this->name, GF_LOG_WARNING, 0,
962 AFR_MSG_EXPUNGING_FILE_OR_DIR,
963 "expunging %s %s/%s on all subvols", type, priv->anon_inode_name,
964 entry->d_name);
965 ret = 0;
966 for (i = 0; i < priv->child_count; i++) {
967 op_errno = -afr_shd_entry_purge(priv->children[i], loc.parent,
968 entry->d_name, iatt->ia_type);
969 if (op_errno != ENOENT && op_errno != ESTALE) {
970 ret |= -op_errno;
971 }
972 }
973 }
974
975 out:
976 if (frame)
977 AFR_STACK_DESTROY(frame);
978 loc_wipe(&loc);
979 return ret;
980 }
981
982 static void
983 afr_cleanup_anon_inode_dir(struct subvol_healer *healer)
984 {
985 int ret = 0;
986 call_frame_t *frame = NULL;
987 afr_private_t *priv = healer->this->private;
988 loc_t loc = {0};
989
990 ret = afr_anon_inode_create(healer->this, healer->subvol, &loc.inode);
991 if (ret)
992 goto out;
993
994 frame = afr_frame_create(healer->this, &ret);
995 if (!frame) {
996 ret = -ret;
997 goto out;
998 }
999
1000 ret = syncop_mt_dir_scan(frame, priv->children[healer->subvol], &loc,
1001 GF_CLIENT_PID_SELF_HEALD, healer,
1002 afr_shd_anon_inode_cleaner, NULL,
1003 priv->shd.max_threads, priv->shd.wait_qlength);
1004 out:
1005 if (frame)
1006 AFR_STACK_DESTROY(frame);
1007 loc_wipe(&loc);
1008 return;
1009 }
1010
1011 void *
1012 afr_shd_index_healer(void *data)
1013 {
1014 struct subvol_healer *healer = NULL;
1015 xlator_t *this = NULL;
1016 int ret = 0;
1017 afr_private_t *priv = NULL;
1018 dict_t *pre_crawl_xdata = NULL;
1019 loc_t loc = {
1020 0,
1021 };
1022 gf_lkowner_t lkowner;
1023 pid_t pid = GF_CLIENT_PID_SELF_HEALD;
1024
1025 healer = data;
1026 THIS = this = healer->this;
1027 priv = this->private;
1028
1029 syncopctx_setfspid(&pid);
1030 set_lk_owner_from_ptr(&lkowner, &lkowner);
1031 syncopctx_setfslkowner(&lkowner);
1032
1033 for (;;) {
1034 afr_shd_healer_wait(healer);
1035
1036 if (!afr_bricks_available_for_heal(priv))
1037 continue;
1038
1039 ASSERT_LOCAL(this, healer);
1040 priv->local[healer->subvol] = healer->local;
1041
1042 if (priv->thin_arbiter_count) {
1043 if (afr_shd_ta_needs_heal(this, healer))
1044 afr_shd_ta_get_xattrs(this, &loc, healer, &pre_crawl_xdata);
1045 }
1046
1047 do {
1048 gf_msg_debug(this->name, 0, "starting index sweep on subvol %s",
1049 afr_subvol_name(this, healer->subvol));
1050
1051 afr_shd_sweep_prepare(healer);
1052
1053 ret = afr_shd_index_sweep_all(healer);
1054
1055 afr_shd_sweep_done(healer);
1056 /*
1057 As long as at least one gfid was
1058 healed, keep retrying. We may have
1059 just healed a directory and thereby
1060 created entries for other gfids which
1061 could not be healed thus far.
1062 */
1063
1064 gf_msg_debug(this->name, 0, "finished index sweep on subvol %s",
1065 afr_subvol_name(this, healer->subvol));
1066 /*
1067 Give a pause before retrying to avoid a busy loop
1068 in case the only entry in index is because of
1069 an ongoing I/O.
1070 */
1071 sleep(1);
1072 } while (ret > 0);
1073
1074 if (ret == 0) {
1075 afr_cleanup_anon_inode_dir(healer);
1076 }
1077
1078 if (ret == 0 && pre_crawl_xdata &&
1079 !healer->crawl_event.heal_failed_count) {
1080 afr_shd_ta_check_and_unset_xattrs(this, &loc, healer,
1081 pre_crawl_xdata);
1082 }
1083
1084 if (pre_crawl_xdata) {
1085 dict_unref(pre_crawl_xdata);
1086 pre_crawl_xdata = NULL;
1087 }
1088 }
1089
1090 return NULL;
1091 }
1092
1093 static void *
1094 afr_shd_full_healer(void *data)
1095 {
1096 struct subvol_healer *healer = NULL;
1097 xlator_t *this = NULL;
1098 int run = 0;
1099
1100 healer = data;
1101 THIS = this = healer->this;
1102
1103 for (;;) {
1104 pthread_mutex_lock(&healer->mutex);
1105 {
1106 run = __afr_shd_healer_wait(healer);
1107 if (!run)
1108 healer->running = _gf_false;
1109 }
1110 pthread_mutex_unlock(&healer->mutex);
1111
1112 if (!run)
1113 break;
1114
1115 ASSERT_LOCAL(this, healer);
1116
1117 gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
1118 "starting full sweep on subvol %s",
1119 afr_subvol_name(this, healer->subvol));
1120
1121 afr_shd_sweep_prepare(healer);
1122
1123 afr_shd_full_sweep(healer, this->itable->root);
1124
1125 afr_shd_sweep_done(healer);
1126
1127 gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
1128 "finished full sweep on subvol %s",
1129 afr_subvol_name(this, healer->subvol));
1130 }
1131
1132 return NULL;
1133 }
1134
1135 static int
1136 afr_shd_healer_init(xlator_t *this, struct subvol_healer *healer)
1137 {
1138 int ret = 0;
1139
1140 ret = pthread_mutex_init(&healer->mutex, NULL);
1141 if (ret)
1142 goto out;
1143
1144 ret = pthread_cond_init(&healer->cond, NULL);
1145 if (ret)
1146 goto out;
1147
1148 healer->this = this;
1149 healer->running = _gf_false;
1150 healer->rerun = _gf_false;
1151 healer->local = _gf_false;
1152 out:
1153 return ret;
1154 }
1155
1156 static int
1157 afr_shd_healer_spawn(xlator_t *this, struct subvol_healer *healer,
1158 void *(threadfn)(void *))
1159 {
1160 int ret = 0;
1161
1162 pthread_mutex_lock(&healer->mutex);
1163 {
1164 if (healer->running) {
1165 pthread_cond_signal(&healer->cond);
1166 } else {
1167 ret = gf_thread_create(&healer->thread, NULL, threadfn, healer,
1168 "shdheal");
1169 if (ret)
1170 goto unlock;
1171 healer->running = 1;
1172 }
1173
1174 healer->rerun = 1;
1175 }
1176 unlock:
1177 pthread_mutex_unlock(&healer->mutex);
1178
1179 return ret;
1180 }
1181
1182 static int
1183 afr_shd_full_healer_spawn(xlator_t *this, int subvol)
1184 {
1185 return afr_shd_healer_spawn(this, NTH_FULL_HEALER(this, subvol),
1186 afr_shd_full_healer);
1187 }
1188
1189 static int
1190 afr_shd_index_healer_spawn(xlator_t *this, int subvol)
1191 {
1192 return afr_shd_healer_spawn(this, NTH_INDEX_HEALER(this, subvol),
1193 afr_shd_index_healer);
1194 }
1195
1196 static int
1197 afr_shd_dict_add_crawl_event(xlator_t *this, dict_t *output,
1198 crawl_event_t *crawl_event)
1199 {
1200 int ret = 0;
1201 uint64_t count = 0;
1202 char key[128] = {0};
1203 int keylen = 0;
1204 char suffix[64] = {0};
1205 int xl_id = 0;
1206 uint64_t healed_count = 0;
1207 uint64_t split_brain_count = 0;
1208 uint64_t heal_failed_count = 0;
1209 char *start_time_str = 0;
1210 char *end_time_str = NULL;
1211 char *crawl_type = NULL;
1212 int progress = -1;
1213 int child = -1;
1214
1215 child = crawl_event->child;
1216 healed_count = crawl_event->healed_count;
1217 split_brain_count = crawl_event->split_brain_count;
1218 heal_failed_count = crawl_event->heal_failed_count;
1219 crawl_type = crawl_event->crawl_type;
1220
1221 if (!crawl_event->start_time)
1222 goto out;
1223
1224 start_time_str = gf_strdup(ctime(&crawl_event->start_time));
1225
1226 if (crawl_event->end_time)
1227 end_time_str = gf_strdup(ctime(&crawl_event->end_time));
1228
1229 ret = dict_get_int32(output, this->name, &xl_id);
1230 if (ret) {
1231 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
1232 "xl does not have id");
1233 goto out;
1234 }
1235
1236 snprintf(key, sizeof(key), "statistics-%d-%d-count", xl_id, child);
1237 ret = dict_get_uint64(output, key, &count);
1238
1239 snprintf(suffix, sizeof(suffix), "%d-%d-%" PRIu64, xl_id, child, count);
1240 snprintf(key, sizeof(key), "statistics_healed_cnt-%s", suffix);
1241 ret = dict_set_uint64(output, key, healed_count);
1242 if (ret) {
1243 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
1244 "Could not add statistics_healed_count to output");
1245 goto out;
1246 }
1247
1248 snprintf(key, sizeof(key), "statistics_sb_cnt-%s", suffix);
1249 ret = dict_set_uint64(output, key, split_brain_count);
1250 if (ret) {
1251 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
1252 "Could not add statistics_split_brain_count to output");
1253 goto out;
1254 }
1255
1256 keylen = snprintf(key, sizeof(key), "statistics_crawl_type-%s", suffix);
1257 ret = dict_set_strn(output, key, keylen, crawl_type);
1258 if (ret) {
1259 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
1260 "Could not add statistics_crawl_type to output");
1261 goto out;
1262 }
1263
1264 snprintf(key, sizeof(key), "statistics_heal_failed_cnt-%s", suffix);
1265 ret = dict_set_uint64(output, key, heal_failed_count);
1266 if (ret) {
1267 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
1268 "Could not add statistics_healed_failed_count to output");
1269 goto out;
1270 }
1271
1272 keylen = snprintf(key, sizeof(key), "statistics_strt_time-%s", suffix);
1273 ret = dict_set_dynstrn(output, key, keylen, start_time_str);
1274 if (ret) {
1275 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
1276 "Could not add statistics_crawl_start_time to output");
1277 goto out;
1278 } else {
1279 start_time_str = NULL;
1280 }
1281
1282 if (!end_time_str)
1283 progress = 1;
1284 else
1285 progress = 0;
1286
1287 keylen = snprintf(key, sizeof(key), "statistics_end_time-%s", suffix);
1288 if (!end_time_str)
1289 end_time_str = gf_strdup("Could not determine the end time");
1290 ret = dict_set_dynstrn(output, key, keylen, end_time_str);
1291 if (ret) {
1292 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
1293 "Could not add statistics_crawl_end_time to output");
1294 goto out;
1295 } else {
1296 end_time_str = NULL;
1297 }
1298
1299 keylen = snprintf(key, sizeof(key), "statistics_inprogress-%s", suffix);
1300
1301 ret = dict_set_int32n(output, key, keylen, progress);
1302 if (ret) {
1303 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
1304 "Could not add statistics_inprogress to output");
1305 goto out;
1306 }
1307
1308 snprintf(key, sizeof(key), "statistics-%d-%d-count", xl_id, child);
1309 ret = dict_set_uint64(output, key, count + 1);
1310 if (ret) {
1311 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
1312 "Could not increment the counter.");
1313 goto out;
1314 }
1315 out:
1316 GF_FREE(start_time_str);
1317 GF_FREE(end_time_str);
1318 return ret;
1319 }
1320
1321 static void
1322 afr_add_crawl_event(circular_buffer_t *cb, void *data)
1323 {
1324 dict_t *output = NULL;
1325 xlator_t *this = THIS;
1326 afr_private_t *priv = NULL;
1327 afr_self_heald_t *shd = NULL;
1328 crawl_event_t *crawl_event = NULL;
1329
1330 output = data;
1331 priv = this->private;
1332 shd = &priv->shd;
1333 crawl_event = cb->data;
1334
1335 if (shd->index_healers[crawl_event->child].local)
1336 afr_shd_dict_add_crawl_event(this, output, crawl_event);
1337 }
1338
1339 int
1340 afr_selfheal_daemon_init(xlator_t *this)
1341 {
1342 afr_private_t *priv = NULL;
1343 afr_self_heald_t *shd = NULL;
1344 int ret = -1;
1345 int i = 0;
1346
1347 priv = this->private;
1348 shd = &priv->shd;
1349
1350 shd->index_healers = GF_CALLOC(sizeof(*shd->index_healers),
1351 priv->child_count,
1352 gf_afr_mt_subvol_healer_t);
1353 if (!shd->index_healers)
1354 goto out;
1355
1356 for (i = 0; i < priv->child_count; i++) {
1357 shd->index_healers[i].subvol = i;
1358 ret = afr_shd_healer_init(this, &shd->index_healers[i]);
1359 if (ret)
1360 goto out;
1361 }
1362
1363 shd->full_healers = GF_CALLOC(sizeof(*shd->full_healers), priv->child_count,
1364 gf_afr_mt_subvol_healer_t);
1365 if (!shd->full_healers)
1366 goto out;
1367 for (i = 0; i < priv->child_count; i++) {
1368 shd->full_healers[i].subvol = i;
1369 ret = afr_shd_healer_init(this, &shd->full_healers[i]);
1370 if (ret)
1371 goto out;
1372 }
1373
1374 shd->split_brain = eh_new(AFR_EH_SPLIT_BRAIN_LIMIT, _gf_false,
1375 afr_destroy_shd_event_data);
1376 if (!shd->split_brain)
1377 goto out;
1378
1379 shd->statistics = GF_CALLOC(sizeof(eh_t *), priv->child_count,
1380 gf_common_mt_eh_t);
1381 if (!shd->statistics)
1382 goto out;
1383
1384 for (i = 0; i < priv->child_count; i++) {
1385 shd->statistics[i] = eh_new(AFR_STATISTICS_HISTORY_SIZE, _gf_false,
1386 afr_destroy_crawl_event_data);
1387 if (!shd->statistics[i])
1388 goto out;
1389 shd->full_healers[i].crawl_event.child = i;
1390 shd->full_healers[i].crawl_event.crawl_type = "FULL";
1391 shd->index_healers[i].crawl_event.child = i;
1392 shd->index_healers[i].crawl_event.crawl_type = "INDEX";
1393 }
1394
1395 ret = 0;
1396 out:
1397 return ret;
1398 }
1399
1400 void
1401 afr_selfheal_childup(xlator_t *this, afr_private_t *priv)
1402 {
1403 int subvol = 0;
1404
1405 if (!priv->shd.iamshd)
1406 return;
1407 for (subvol = 0; subvol < priv->child_count; subvol++)
1408 if (priv->child_up[subvol])
1409 afr_shd_index_healer_spawn(this, subvol);
1410
1411 return;
1412 }
1413
1414 static int
1415 afr_shd_get_index_count(xlator_t *this, int i, uint64_t *count)
1416 {
1417 afr_private_t *priv = NULL;
1418 xlator_t *subvol = NULL;
1419 loc_t rootloc = {
1420 0,
1421 };
1422 dict_t *xattr = NULL;
1423 int ret = -1;
1424
1425 priv = this->private;
1426 subvol = priv->children[i];
1427
1428 rootloc.inode = inode_ref(this->itable->root);
1429 gf_uuid_copy(rootloc.gfid, rootloc.inode->gfid);
1430
1431 ret = syncop_getxattr(subvol, &rootloc, &xattr, GF_XATTROP_INDEX_COUNT,
1432 NULL, NULL);
1433 if (ret < 0)
1434 goto out;
1435
1436 ret = dict_get_uint64(xattr, GF_XATTROP_INDEX_COUNT, count);
1437 if (ret)
1438 goto out;
1439
1440 ret = 0;
1441
1442 out:
1443 if (xattr)
1444 dict_unref(xattr);
1445 loc_wipe(&rootloc);
1446
1447 return ret;
1448 }
1449
1450 int
1451 afr_xl_op(xlator_t *this, dict_t *input, dict_t *output)
1452 {
1453 gf_xl_afr_op_t op = GF_SHD_OP_INVALID;
1454 int ret = 0;
1455 int xl_id = 0;
1456 afr_private_t *priv = NULL;
1457 afr_self_heald_t *shd = NULL;
1458 struct subvol_healer *healer = NULL;
1459 int i = 0;
1460 char key[64];
1461 int keylen = 0;
1462 int this_name_len = 0;
1463 int op_ret = 0;
1464 uint64_t cnt = 0;
1465
1466 #define AFR_SET_DICT_AND_LOG(name, output, key, keylen, dict_str, \
1467 dict_str_len) \
1468 { \
1469 int ret; \
1470 \
1471 ret = dict_set_nstrn(output, key, keylen, dict_str, dict_str_len); \
1472 if (ret) { \
1473 gf_smsg(name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED, \
1474 "key=%s", key, "value=%s", dict_str, NULL); \
1475 } \
1476 }
1477
1478 priv = this->private;
1479 shd = &priv->shd;
1480
1481 ret = dict_get_int32_sizen(input, "xl-op", (int32_t *)&op);
1482 if (ret) {
1483 gf_smsg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
1484 "key=xl-op", NULL);
1485 goto out;
1486 }
1487 this_name_len = strlen(this->name);
1488 ret = dict_get_int32n(input, this->name, this_name_len, &xl_id);
1489 if (ret) {
1490 gf_smsg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
1491 "key=%s", this->name, NULL);
1492 goto out;
1493 }
1494 ret = dict_set_int32n(output, this->name, this_name_len, xl_id);
1495 if (ret) {
1496 gf_smsg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
1497 "key=%s", this->name, NULL);
1498 goto out;
1499 }
1500 switch (op) {
1501 case GF_SHD_OP_HEAL_INDEX:
1502 op_ret = 0;
1503
1504 for (i = 0; i < priv->child_count; i++) {
1505 healer = &shd->index_healers[i];
1506 keylen = snprintf(key, sizeof(key), "%d-%d-status", xl_id, i);
1507
1508 if (!priv->child_up[i]) {
1509 AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
1510 SBRICK_NOT_CONNECTED,
1511 SLEN(SBRICK_NOT_CONNECTED));
1512 op_ret = -1;
1513 } else if (AFR_COUNT(priv->child_up, priv->child_count) < 2) {
1514 AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
1515 SLESS_THAN2_BRICKS_in_REP,
1516 SLEN(SLESS_THAN2_BRICKS_in_REP));
1517 op_ret = -1;
1518 } else if (!afr_shd_is_subvol_local(this, healer->subvol)) {
1519 AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
1520 SBRICK_IS_REMOTE,
1521 SLEN(SBRICK_IS_REMOTE));
1522 } else {
1523 AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
1524 SSTARTED_SELF_HEAL,
1525 SLEN(SSTARTED_SELF_HEAL));
1526
1527 ret = afr_shd_index_healer_spawn(this, i);
1528
1529 if (ret) {
1530 gf_smsg(this->name, GF_LOG_ERROR, -ret,
1531 AFR_MSG_HEALER_SPAWN_FAILED, NULL);
1532 }
1533 }
1534 }
1535 break;
1536 case GF_SHD_OP_HEAL_FULL:
1537 op_ret = -1;
1538
1539 for (i = 0; i < priv->child_count; i++) {
1540 healer = &shd->full_healers[i];
1541 keylen = snprintf(key, sizeof(key), "%d-%d-status", xl_id, i);
1542
1543 if (!priv->child_up[i]) {
1544 AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
1545 SBRICK_NOT_CONNECTED,
1546 SLEN(SBRICK_NOT_CONNECTED));
1547 } else if (AFR_COUNT(priv->child_up, priv->child_count) < 2) {
1548 AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
1549 SLESS_THAN2_BRICKS_in_REP,
1550 SLEN(SLESS_THAN2_BRICKS_in_REP));
1551 } else if (!afr_shd_is_subvol_local(this, healer->subvol)) {
1552 AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
1553 SBRICK_IS_REMOTE,
1554 SLEN(SBRICK_IS_REMOTE));
1555 } else {
1556 AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
1557 SSTARTED_SELF_HEAL,
1558 SLEN(SSTARTED_SELF_HEAL));
1559
1560 ret = afr_shd_full_healer_spawn(this, i);
1561
1562 if (ret) {
1563 gf_smsg(this->name, GF_LOG_ERROR, -ret,
1564 AFR_MSG_HEALER_SPAWN_FAILED, NULL);
1565 }
1566 op_ret = 0;
1567 }
1568 }
1569 break;
1570 case GF_SHD_OP_INDEX_SUMMARY:
1571 case GF_SHD_OP_SPLIT_BRAIN_FILES:
1572 /* this case has been handled in glfs-heal.c */
1573 break;
1574 case GF_SHD_OP_STATISTICS:
1575 for (i = 0; i < priv->child_count; i++) {
1576 eh_dump(shd->statistics[i], output, afr_add_crawl_event);
1577 ret = afr_shd_dict_add_crawl_event(
1578 this, output, &shd->index_healers[i].crawl_event);
1579 if (ret) {
1580 gf_smsg(this->name, GF_LOG_ERROR, -ret,
1581 AFR_MSG_ADD_CRAWL_EVENT_FAILED, NULL);
1582 }
1583
1584 ret = afr_shd_dict_add_crawl_event(
1585 this, output, &shd->full_healers[i].crawl_event);
1586 if (ret) {
1587 gf_smsg(this->name, GF_LOG_ERROR, -ret,
1588 AFR_MSG_ADD_CRAWL_EVENT_FAILED, NULL);
1589 }
1590 }
1591 break;
1592 case GF_SHD_OP_STATISTICS_HEAL_COUNT:
1593 case GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
1594 op_ret = -1;
1595
1596 for (i = 0; i < priv->child_count; i++) {
1597 if (!priv->child_up[i]) {
1598 keylen = snprintf(key, sizeof(key), "%d-%d-status", xl_id,
1599 i);
1600 AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
1601 SBRICK_NOT_CONNECTED,
1602 SLEN(SBRICK_NOT_CONNECTED));
1603 } else {
1604 snprintf(key, sizeof(key), "%d-%d-hardlinks", xl_id, i);
1605 ret = afr_shd_get_index_count(this, i, &cnt);
1606 if (ret == 0) {
1607 ret = dict_set_uint64(output, key, cnt);
1608 }
1609 if (ret) {
1610 gf_smsg(this->name, GF_LOG_ERROR, -ret,
1611 AFR_MSG_DICT_SET_FAILED, NULL);
1612 }
1613 op_ret = 0;
1614 }
1615 }
1616
1617 break;
1618
1619 default:
1620 gf_smsg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_ARG, "op=%d",
1621 op, NULL);
1622 break;
1623 }
1624 out:
1625 dict_deln(output, this->name, this_name_len);
1626 return op_ret;
1627
1628 #undef AFR_SET_DICT_AND_LOG
1629 }