Debian Codegraph @

xlators/cluster/afr/src/afr-self-heal-entry.c

raw
1 /*
2 Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
3 This file is part of GlusterFS.
4
5 This file is licensed to you under your choice of the GNU Lesser
6 General Public License, version 3 or any later version (LGPLv3 or
7 later), or the GNU General Public License, version 2 (GPLv2), in all
8 cases as published by the Free Software Foundation.
9 */
10
11 #include "afr.h"
12 #include "afr-self-heal.h"
13 #include "afr-transaction.h"
14 #include "afr-messages.h"
15 #include <glusterfs/syncop-utils.h>
16 #include <glusterfs/events.h>
17
18 static int
19 afr_selfheal_entry_anon_inode(xlator_t *this, inode_t *dir, const char *name,
20 inode_t *inode, int child,
21 struct afr_reply *replies,
22 gf_boolean_t *anon_inode)
23 {
24 afr_private_t *priv = NULL;
25 afr_local_t *local = NULL;
26 xlator_t *subvol = NULL;
27 int ret = 0;
28 int i = 0;
29 char g[64] = {0};
30 unsigned char *lookup_success = NULL;
31 call_frame_t *frame = NULL;
32 loc_t loc2 = {
33 0,
34 };
35 loc_t loc = {
36 0,
37 };
38
39 priv = this->private;
40 subvol = priv->children[child];
41 lookup_success = alloca0(priv->child_count);
42 uuid_utoa_r(replies[child].poststat.ia_gfid, g);
43 loc.inode = inode_new(inode->table);
44 if (!loc.inode) {
45 ret = -ENOMEM;
46 goto out;
47 }
48
49 if (replies[child].poststat.ia_type == IA_IFDIR) {
50 /* This directory may have sub-directory hierarchy which may need to
51 * be preserved for subsequent heals. So unconditionally move the
52 * directory to anonymous-inode directory*/
53 *anon_inode = _gf_true;
54 goto anon_inode;
55 }
56
57 frame = afr_frame_create(this, &ret);
58 if (!frame) {
59 ret = -ret;
60 goto out;
61 }
62 local = frame->local;
63 gf_uuid_copy(loc.gfid, replies[child].poststat.ia_gfid);
64 AFR_ONLIST(local->child_up, frame, afr_selfheal_discover_cbk, lookup, &loc,
65 NULL);
66 for (i = 0; i < priv->child_count; i++) {
67 if (local->replies[i].op_ret == 0) {
68 lookup_success[i] = 1;
69 } else if (local->replies[i].op_errno != ENOENT &&
70 local->replies[i].op_errno != ESTALE) {
71 ret = -local->replies[i].op_errno;
72 }
73 }
74
75 if (priv->quorum_count) {
76 if (afr_has_quorum(lookup_success, priv, NULL)) {
77 *anon_inode = _gf_true;
78 }
79 } else if (AFR_COUNT(lookup_success, priv->child_count) > 1) {
80 *anon_inode = _gf_true;
81 } else if (ret) {
82 goto out;
83 }
84
85 anon_inode:
86 if (!*anon_inode) {
87 ret = 0;
88 goto out;
89 }
90
91 loc.parent = inode_ref(dir);
92 gf_uuid_copy(loc.pargfid, dir->gfid);
93 loc.name = name;
94
95 ret = afr_anon_inode_create(this, child, &loc2.parent);
96 if (ret < 0)
97 goto out;
98
99 loc2.name = g;
100 ret = syncop_rename(subvol, &loc, &loc2, NULL, NULL);
101 if (ret < 0) {
102 gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_EXPUNGING_FILE_OR_DIR,
103 "Rename to %s dir %s/%s (%s) on %s failed",
104 priv->anon_inode_name, uuid_utoa(dir->gfid), name, g,
105 subvol->name);
106 } else {
107 gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_EXPUNGING_FILE_OR_DIR,
108 "Rename to %s dir %s/%s (%s) on %s successful",
109 priv->anon_inode_name, uuid_utoa(dir->gfid), name, g,
110 subvol->name);
111 }
112
113 out:
114 loc_wipe(&loc);
115 loc_wipe(&loc2);
116 if (frame) {
117 AFR_STACK_DESTROY(frame);
118 }
119
120 return ret;
121 }
122
123 int
124 afr_selfheal_entry_delete(xlator_t *this, inode_t *dir, const char *name,
125 inode_t *inode, int child, struct afr_reply *replies)
126 {
127 char g[64] = {0};
128 afr_private_t *priv = NULL;
129 xlator_t *subvol = NULL;
130 int ret = 0;
131 loc_t loc = {
132 0,
133 };
134 gf_boolean_t anon_inode = _gf_false;
135
136 priv = this->private;
137 subvol = priv->children[child];
138
139 if ((!replies[child].valid) || (replies[child].op_ret < 0)) {
140 /*Nothing to do*/
141 ret = 0;
142 goto out;
143 }
144
145 if (priv->use_anon_inode) {
146 ret = afr_selfheal_entry_anon_inode(this, dir, name, inode, child,
147 replies, &anon_inode);
148 if (ret < 0 || anon_inode)
149 goto out;
150 }
151
152 loc.parent = inode_ref(dir);
153 loc.inode = inode_new(inode->table);
154 if (!loc.inode) {
155 ret = -ENOMEM;
156 goto out;
157 }
158 loc.name = name;
159 switch (replies[child].poststat.ia_type) {
160 case IA_IFDIR:
161 gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_EXPUNGING_FILE_OR_DIR,
162 "expunging dir %s/%s (%s) on %s", uuid_utoa(dir->gfid), name,
163 uuid_utoa_r(replies[child].poststat.ia_gfid, g),
164 subvol->name);
165 ret = syncop_rmdir(subvol, &loc, 1, NULL, NULL);
166 break;
167 default:
168 gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_EXPUNGING_FILE_OR_DIR,
169 "expunging file %s/%s (%s) on %s", uuid_utoa(dir->gfid),
170 name, uuid_utoa_r(replies[child].poststat.ia_gfid, g),
171 subvol->name);
172 ret = syncop_unlink(subvol, &loc, NULL, NULL);
173 break;
174 }
175
176 out:
177 loc_wipe(&loc);
178 return ret;
179 }
180
181 static int
182 afr_new_entry_mark_status(call_frame_t *frame, loc_t *loc,
183 struct afr_reply *lookup_replies,
184 unsigned char *sources, int source, int dst,
185 gf_boolean_t *dst_hardlink)
186 {
187 xlator_t *this = frame->this;
188 afr_private_t *priv = this->private;
189 struct iatt *iatt = NULL;
190 int pending = 0;
191 int metadata_idx = 0;
192 int idx = -1;
193 int ret = 0;
194 int i = 0;
195
196 if (source == -1) {
197 goto lookup;
198 }
199
200 iatt = &lookup_replies[source].poststat;
201
202 if (IA_ISDIR(lookup_replies[source].poststat.ia_type)) {
203 goto lookup;
204 }
205
206 metadata_idx = afr_index_for_transaction_type(AFR_METADATA_TRANSACTION);
207 if (IA_ISREG(lookup_replies[source].poststat.ia_type)) {
208 idx = afr_index_for_transaction_type(AFR_DATA_TRANSACTION);
209 }
210
211 for (i = 0; i < priv->child_count; i++) {
212 if (!sources[i]) {
213 continue;
214 }
215
216 if (uuid_compare(lookup_replies[i].poststat.ia_gfid, loc->gfid)) {
217 /*Future proofing this call's input arguments*/
218 GF_ASSERT(!"gfids don't match for call to new entry marking");
219 goto lookup;
220 }
221 afr_selfheal_fill_cell(frame->this->private, lookup_replies[i].xdata,
222 &pending, dst, metadata_idx);
223 if (pending == 0) {
224 goto lookup;
225 }
226
227 if (idx == -1) {
228 continue;
229 }
230
231 afr_selfheal_fill_cell(frame->this->private, lookup_replies[i].xdata,
232 &pending, dst, idx);
233 if (pending == 0) {
234 goto lookup;
235 }
236 }
237 if (iatt->ia_type == IA_IFLNK && dst_hardlink) {
238 ret = syncop_lookup(priv->children[dst], loc, 0, 0, 0, 0);
239 if (ret == 0) {
240 *dst_hardlink = _gf_true;
241 }
242 /* If it is a soft link, we need to check if a hardlink to
243 * this softlink present in the dst, hence we perform a
244 * lookup here*/
245 }
246 /*Pending is marked on all source bricks, we definitely know that new entry
247 * marking is not needed*/
248 return 0;
249
250 lookup:
251 ret = syncop_lookup(priv->children[dst], loc, 0, 0, 0, 0);
252 if (ret == 0 && dst_hardlink) {
253 *dst_hardlink = _gf_true;
254 }
255
256 return ret;
257 }
258
259 int
260 afr_selfheal_recreate_entry(call_frame_t *frame, int dst, int source,
261 unsigned char *sources, inode_t *dir,
262 const char *name, inode_t *inode,
263 struct afr_reply *replies)
264 {
265 int ret = 0;
266 loc_t loc = {
267 0,
268 };
269 loc_t srcloc = {
270 0,
271 };
272 loc_t anonloc = {
273 0,
274 };
275 xlator_t *this = frame->this;
276 afr_private_t *priv = NULL;
277 dict_t *xdata = NULL;
278 struct iatt *iatt = NULL;
279 char *linkname = NULL;
280 mode_t mode = 0;
281 struct iatt newent = {
282 0,
283 };
284 unsigned char *newentry = NULL;
285 char iatt_uuid_str[64] = {0};
286 char dir_uuid_str[64] = {0};
287 gf_boolean_t dst_hardlink = _gf_false;
288
289 priv = this->private;
290 iatt = &replies[source].poststat;
291 uuid_utoa_r(iatt->ia_gfid, iatt_uuid_str);
292 if (iatt->ia_type == IA_INVAL || gf_uuid_is_null(iatt->ia_gfid)) {
293 gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SELF_HEAL_FAILED,
294 "Invalid ia_type (%d) or gfid(%s). source brick=%d, "
295 "pargfid=%s, name=%s",
296 iatt->ia_type, iatt_uuid_str, source,
297 uuid_utoa_r(dir->gfid, dir_uuid_str), name);
298 ret = -EINVAL;
299 goto out;
300 }
301
302 xdata = dict_new();
303 if (!xdata)
304 return -ENOMEM;
305 newentry = alloca0(priv->child_count);
306 loc.parent = inode_ref(dir);
307 gf_uuid_copy(loc.pargfid, dir->gfid);
308 loc.name = name;
309 loc.inode = inode_ref(inode);
310
311 ret = afr_selfheal_entry_delete(this, dir, name, inode, dst, replies);
312 if (ret)
313 goto out;
314
315 ret = dict_set_gfuuid(xdata, "gfid-req", replies[source].poststat.ia_gfid,
316 true);
317 if (ret)
318 goto out;
319
320 srcloc.inode = inode_ref(inode);
321 gf_uuid_copy(srcloc.gfid, iatt->ia_gfid);
322 ret = afr_new_entry_mark_status(frame, &srcloc, replies, sources, source,
323 dst, &dst_hardlink);
324 if (ret == -ENOENT || ret == -ESTALE) {
325 newentry[dst] = 1;
326 ret = afr_selfheal_newentry_mark(frame, this, inode, source, replies,
327 sources, newentry);
328 if (ret)
329 goto out;
330 } else if (ret == 0 && iatt->ia_type == IA_IFDIR && priv->use_anon_inode) {
331 // Try rename from hidden directory
332 ret = afr_anon_inode_create(this, dst, &anonloc.parent);
333 if (ret < 0)
334 goto out;
335 anonloc.inode = inode_ref(inode);
336 anonloc.name = iatt_uuid_str;
337 ret = syncop_rename(priv->children[dst], &anonloc, &loc, NULL, NULL);
338 if (ret == -ENOENT || ret == -ESTALE)
339 ret = -1; /*This sets 'mismatch' to true*/
340 goto out;
341 }
342
343 mode = st_mode_from_ia(iatt->ia_prot, iatt->ia_type);
344
345 switch (iatt->ia_type) {
346 case IA_IFDIR:
347 ret = syncop_mkdir(priv->children[dst], &loc, mode, 0, xdata, NULL);
348 break;
349 case IA_IFLNK:
350 if (dst_hardlink) {
351 ret = syncop_link(priv->children[dst], &srcloc, &loc, &newent,
352 NULL, NULL);
353 } else {
354 ret = syncop_readlink(priv->children[source], &srcloc,
355 &linkname, 4096, NULL, NULL);
356 if (ret <= 0)
357 goto out;
358 ret = syncop_symlink(priv->children[dst], &loc, linkname, NULL,
359 xdata, NULL);
360 }
361 break;
362 default:
363 ret = dict_set_int32_sizen(xdata, GLUSTERFS_INTERNAL_FOP_KEY, 1);
364 if (ret)
365 goto out;
366 ret = syncop_mknod(
367 priv->children[dst], &loc, mode,
368 makedev(ia_major(iatt->ia_rdev), ia_minor(iatt->ia_rdev)),
369 &newent, xdata, NULL);
370 break;
371 }
372
373 out:
374 if (xdata)
375 dict_unref(xdata);
376 GF_FREE(linkname);
377 loc_wipe(&loc);
378 loc_wipe(&srcloc);
379 loc_wipe(&anonloc);
380 return ret;
381 }
382
383 static int
384 __afr_selfheal_heal_dirent(call_frame_t *frame, xlator_t *this, fd_t *fd,
385 char *name, inode_t *inode, int source,
386 unsigned char *sources, unsigned char *healed_sinks,
387 unsigned char *locked_on, struct afr_reply *replies)
388 {
389 int ret = 0;
390 afr_private_t *priv = NULL;
391 int i = 0;
392
393 priv = this->private;
394
395 if (!replies[source].valid)
396 return -EIO;
397
398 /* Skip healing this entry if the last lookup on it failed for reasons
399 * other than ENOENT.
400 */
401 if ((replies[source].op_ret < 0) && (replies[source].op_errno != ENOENT))
402 return -replies[source].op_errno;
403
404 if (replies[source].op_ret == 0) {
405 ret = afr_lookup_and_heal_gfid(this, fd->inode, name, inode, replies,
406 source, sources,
407 &replies[source].poststat.ia_gfid, NULL);
408 if (ret)
409 return ret;
410 }
411
412 for (i = 0; i < priv->child_count; i++) {
413 if (!healed_sinks[i])
414 continue;
415 if (replies[source].op_ret == -1 &&
416 replies[source].op_errno == ENOENT) {
417 ret = afr_selfheal_entry_delete(this, fd->inode, name, inode, i,
418 replies);
419 } else {
420 if (!gf_uuid_compare(replies[i].poststat.ia_gfid,
421 replies[source].poststat.ia_gfid)) {
422 gf_msg_debug(this->name, 0, "skipping %s, no heal needed.",
423 name);
424 continue;
425 }
426
427 ret = afr_selfheal_recreate_entry(frame, i, source, sources,
428 fd->inode, name, inode, replies);
429 }
430 if (ret < 0)
431 break;
432 }
433
434 return ret;
435 }
436
437 static int
438 afr_selfheal_detect_gfid_and_type_mismatch(xlator_t *this,
439 struct afr_reply *replies,
440 inode_t *inode, uuid_t pargfid,
441 char *bname, int src_idx,
442 unsigned char *locked_on, int *src)
443 {
444 int i = 0;
445 int ret = -1;
446 afr_private_t *priv = NULL;
447 void *gfid = NULL;
448 ia_type_t ia_type = IA_INVAL;
449
450 priv = this->private;
451 gfid = &replies[src_idx].poststat.ia_gfid;
452 ia_type = replies[src_idx].poststat.ia_type;
453
454 for (i = 0; i < priv->child_count; i++) {
455 if (i == src_idx)
456 continue;
457
458 if (!replies[i].valid)
459 continue;
460
461 if (replies[i].op_ret != 0)
462 continue;
463
464 if (gf_uuid_is_null(replies[i].poststat.ia_gfid))
465 continue;
466
467 if (replies[i].poststat.ia_type == IA_INVAL)
468 continue;
469
470 if (ia_type == IA_INVAL || gf_uuid_is_null(gfid)) {
471 src_idx = i;
472 ia_type = replies[src_idx].poststat.ia_type;
473 gfid = &replies[src_idx].poststat.ia_gfid;
474 continue;
475 }
476
477 if (gf_uuid_compare(gfid, replies[i].poststat.ia_gfid) &&
478 (ia_type == replies[i].poststat.ia_type)) {
479 ret = afr_gfid_split_brain_source(this, replies, inode, pargfid,
480 bname, src_idx, i, locked_on, src,
481 NULL, NULL);
482 if (ret)
483 gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
484 "Skipping conservative merge on the "
485 "file.");
486 return ret;
487 }
488
489 if (ia_type != replies[i].poststat.ia_type) {
490 gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
491 "Type mismatch detected "
492 "for <gfid:%s>/%s>, %s on %s and %s on %s. "
493 "Skipping conservative merge on the file.",
494 uuid_utoa(pargfid), bname,
495 gf_inode_type_to_str(replies[i].poststat.ia_type),
496 priv->children[i]->name,
497 gf_inode_type_to_str(replies[src_idx].poststat.ia_type),
498 priv->children[src_idx]->name);
499 gf_event(EVENT_AFR_SPLIT_BRAIN,
500 "client-pid=%d;"
501 "subvol=%s;type=file;"
502 "file=<gfid:%s>/%s>;count=2;child-%d=%s;type-"
503 "%d=%s;child-%d=%s;type-%d=%s",
504 this->ctx->cmd_args.client_pid, this->name,
505 uuid_utoa(pargfid), bname, i, priv->children[i]->name, i,
506 gf_inode_type_to_str(replies[i].poststat.ia_type), src_idx,
507 priv->children[src_idx]->name, src_idx,
508 gf_inode_type_to_str(replies[src_idx].poststat.ia_type));
509 return -1;
510 }
511 }
512
513 return 0;
514 }
515
516 static int
517 __afr_selfheal_merge_dirent(call_frame_t *frame, xlator_t *this, fd_t *fd,
518 char *name, inode_t *inode, unsigned char *sources,
519 unsigned char *healed_sinks,
520 unsigned char *locked_on, struct afr_reply *replies)
521 {
522 int ret = 0;
523 int i = 0;
524 int source = -1;
525 int src = -1;
526 afr_private_t *priv = NULL;
527
528 priv = this->private;
529
530 for (i = 0; i < priv->child_count; i++) {
531 if (replies[i].valid && replies[i].op_ret == 0) {
532 source = i;
533 break;
534 }
535 }
536
537 if (source == -1) {
538 /* entry got deleted in the mean time? */
539 return 0;
540 }
541
542 /* Set all the sources as 1, otheriwse newentry_mark won't be set */
543 for (i = 0; i < priv->child_count; i++) {
544 if (replies[i].valid && replies[i].op_ret == 0) {
545 sources[i] = 1;
546 }
547 }
548
549 ret = afr_lookup_and_heal_gfid(this, fd->inode, name, inode, replies,
550 source, sources,
551 &replies[source].poststat.ia_gfid, NULL);
552 if (ret)
553 return ret;
554
555 /* In case of type mismatch / unable to resolve gfid mismatch on the
556 * entry, return -EIO.*/
557 ret = afr_selfheal_detect_gfid_and_type_mismatch(
558 this, replies, inode, fd->inode->gfid, name, source, locked_on, &src);
559
560 if (ret < 0)
561 return ret;
562 if (src != -1) {
563 source = src;
564 for (i = 0; i < priv->child_count; i++) {
565 if (i != src && replies[i].valid &&
566 gf_uuid_compare(replies[src].poststat.ia_gfid,
567 replies[i].poststat.ia_gfid)) {
568 sources[i] = 0;
569 }
570 }
571 }
572
573 for (i = 0; i < priv->child_count; i++) {
574 if (i == source || !healed_sinks[i])
575 continue;
576
577 if (src != -1) {
578 if (!gf_uuid_compare(replies[src].poststat.ia_gfid,
579 replies[i].poststat.ia_gfid))
580 continue;
581 } else if (replies[i].op_errno != ENOENT) {
582 continue;
583 }
584
585 ret |= afr_selfheal_recreate_entry(frame, i, source, sources, fd->inode,
586 name, inode, replies);
587 }
588
589 return ret;
590 }
591
592 static int
593 __afr_selfheal_entry_dirent(call_frame_t *frame, xlator_t *this, fd_t *fd,
594 char *name, inode_t *inode, int source,
595 unsigned char *sources, unsigned char *healed_sinks,
596 unsigned char *locked_on, struct afr_reply *replies)
597 {
598 int ret = -1;
599
600 if (source < 0)
601 ret = __afr_selfheal_merge_dirent(frame, this, fd, name, inode, sources,
602 healed_sinks, locked_on, replies);
603 else
604 ret = __afr_selfheal_heal_dirent(frame, this, fd, name, inode, source,
605 sources, healed_sinks, locked_on,
606 replies);
607 return ret;
608 }
609
610 static gf_boolean_t
611 is_full_heal_marker_present(xlator_t *this, dict_t *xdata, int idx)
612 {
613 int i = 0;
614 void *pending_raw = NULL;
615 afr_private_t *priv = NULL;
616
617 priv = this->private;
618
619 if (!xdata)
620 return _gf_false;
621
622 /* Iterate over each of the priv->pending_keys[] elements and then
623 * see if any of them have data segment non-zero. If they do, return
624 * true. Else return false.
625 */
626 for (i = 0; i < priv->child_count; i++) {
627 if (dict_get_ptr(xdata, priv->pending_key[i], &pending_raw))
628 continue;
629
630 if (!pending_raw)
631 continue;
632
633 if (*((int *)pending_raw + idx))
634 return _gf_true;
635 }
636
637 return _gf_false;
638 }
639
640 static gf_boolean_t
641 afr_need_full_heal(xlator_t *this, struct afr_reply *replies, int source,
642 unsigned char *healed_sinks, afr_transaction_type type)
643 {
644 int i = 0;
645 int idx = 0;
646 afr_private_t *priv = NULL;
647
648 priv = this->private;
649
650 if (!priv->esh_granular)
651 return _gf_true;
652
653 if (type != AFR_ENTRY_TRANSACTION)
654 return _gf_true;
655
656 priv = this->private;
657 idx = afr_index_for_transaction_type(AFR_DATA_TRANSACTION);
658
659 /* If there is a clear source, check whether the full-heal-indicator
660 * is present in its xdata. Otherwise, we need to examine all the
661 * participating bricks and then figure if *even* one of them has a
662 * full-heal-indicator.
663 */
664
665 if (source != -1) {
666 if (is_full_heal_marker_present(this, replies[source].xdata, idx))
667 return _gf_true;
668 }
669
670 /* else ..*/
671
672 for (i = 0; i < priv->child_count; i++) {
673 if (!healed_sinks[i])
674 continue;
675
676 if (is_full_heal_marker_present(this, replies[i].xdata, idx))
677 return _gf_true;
678 }
679
680 return _gf_false;
681 }
682
683 static int
684 __afr_selfheal_entry_finalize_source(xlator_t *this, unsigned char *sources,
685 unsigned char *healed_sinks,
686 unsigned char *locked_on,
687 struct afr_reply *replies,
688 uint64_t *witness)
689 {
690 afr_private_t *priv = NULL;
691 int source = -1;
692 int sources_count = 0;
693 int i = 0;
694
695 priv = this->private;
696
697 sources_count = AFR_COUNT(sources, priv->child_count);
698
699 if ((AFR_CMP(locked_on, healed_sinks, priv->child_count) == 0) ||
700 !sources_count || afr_does_witness_exist(this, witness)) {
701 memset(sources, 0, sizeof(*sources) * priv->child_count);
702 afr_mark_active_sinks(this, sources, locked_on, healed_sinks);
703 return -1;
704 }
705
706 source = afr_choose_source_by_policy(priv, sources, AFR_ENTRY_TRANSACTION);
707
708 /*If the selected source does not blame any other brick, then mark
709 * everything as sink to trigger conservative merge.
710 */
711 if (source != -1 && !AFR_COUNT(healed_sinks, priv->child_count)) {
712 for (i = 0; i < priv->child_count; i++) {
713 if (locked_on[i]) {
714 sources[i] = 0;
715 healed_sinks[i] = 1;
716 }
717 }
718 return -1;
719 }
720
721 return source;
722 }
723
724 int
725 __afr_selfheal_entry_prepare(call_frame_t *frame, xlator_t *this,
726 inode_t *inode, unsigned char *locked_on,
727 unsigned char *sources, unsigned char *sinks,
728 unsigned char *healed_sinks,
729 struct afr_reply *replies, int *source_p,
730 unsigned char *pflag)
731 {
732 int ret = -1;
733 int source = -1;
734 afr_private_t *priv = NULL;
735 uint64_t *witness = NULL;
736
737 priv = this->private;
738
739 ret = afr_selfheal_unlocked_discover(frame, inode, inode->gfid, replies);
740 if (ret)
741 return ret;
742
743 witness = alloca0(sizeof(*witness) * priv->child_count);
744 ret = afr_selfheal_find_direction(frame, this, replies,
745 AFR_ENTRY_TRANSACTION, locked_on, sources,
746 sinks, witness, pflag);
747 if (ret)
748 return ret;
749
750 /* Initialize the healed_sinks[] array optimistically to
751 the intersection of to-be-healed (i.e sinks[]) and
752 the list of servers which are up (i.e locked_on[]).
753
754 As we encounter failures in the healing process, we
755 will unmark the respective servers in the healed_sinks[]
756 array.
757 */
758 AFR_INTERSECT(healed_sinks, sinks, locked_on, priv->child_count);
759
760 source = __afr_selfheal_entry_finalize_source(this, sources, healed_sinks,
761 locked_on, replies, witness);
762
763 if (source < 0) {
764 /* If source is < 0 (typically split-brain), we perform a
765 conservative merge of entries rather than erroring out */
766 }
767 *source_p = source;
768
769 return ret;
770 }
771
772 static int
773 afr_selfheal_entry_dirent(call_frame_t *frame, xlator_t *this, fd_t *fd,
774 char *name, inode_t *parent_idx_inode,
775 xlator_t *subvol, gf_boolean_t full_crawl)
776 {
777 int ret = 0;
778 int source = -1;
779 unsigned char *locked_on = NULL;
780 unsigned char *sources = NULL;
781 unsigned char *sinks = NULL;
782 unsigned char *healed_sinks = NULL;
783 inode_t *inode = NULL;
784 struct afr_reply *replies = NULL;
785 struct afr_reply *par_replies = NULL;
786 afr_private_t *priv = NULL;
787 dict_t *xattr = NULL;
788
789 priv = this->private;
790
791 if (__is_root_gfid(fd->inode->gfid) &&
792 afr_is_private_directory(priv, name, GF_CLIENT_PID_SELF_HEALD)) {
793 return 0;
794 }
795
796 xattr = dict_new();
797 if (!xattr)
798 return -ENOMEM;
799 ret = dict_set_int32_sizen(xattr, GF_GFIDLESS_LOOKUP, 1);
800 if (ret) {
801 dict_unref(xattr);
802 return -1;
803 }
804
805 sources = alloca0(priv->child_count);
806 sinks = alloca0(priv->child_count);
807 healed_sinks = alloca0(priv->child_count);
808 locked_on = alloca0(priv->child_count);
809
810 replies = alloca0(priv->child_count * sizeof(*replies));
811 par_replies = alloca0(priv->child_count * sizeof(*par_replies));
812
813 ret = afr_selfheal_entrylk(frame, this, fd->inode, this->name, NULL,
814 locked_on);
815 {
816 if (ret < priv->child_count) {
817 gf_msg_debug(this->name, 0,
818 "%s: Skipping "
819 "entry self-heal as only %d sub-volumes "
820 " could be locked in %s domain",
821 uuid_utoa(fd->inode->gfid), ret, this->name);
822 ret = -ENOTCONN;
823 goto unlock;
824 }
825
826 ret = __afr_selfheal_entry_prepare(frame, this, fd->inode, locked_on,
827 sources, sinks, healed_sinks,
828 par_replies, &source, NULL);
829 if (ret < 0)
830 goto unlock;
831
832 inode = afr_selfheal_unlocked_lookup_on(frame, fd->inode, name, replies,
833 locked_on, xattr);
834 if (!inode) {
835 ret = -ENOMEM;
836 goto unlock;
837 }
838
839 ret = __afr_selfheal_entry_dirent(frame, this, fd, name, inode, source,
840 sources, healed_sinks, locked_on,
841 replies);
842
843 if ((ret == 0) && (priv->esh_granular) && parent_idx_inode) {
844 ret = afr_shd_entry_purge(subvol, parent_idx_inode, name,
845 inode->ia_type);
846 /* Why is ret force-set to 0? We do not care about
847 * index purge failing for full heal as it is quite
848 * possible during replace-brick that not all files
849 * and directories have their name indices present in
850 * entry-changes/.
851 */
852 ret = 0;
853 }
854 }
855
856 unlock:
857 afr_selfheal_unentrylk(frame, this, fd->inode, this->name, NULL, locked_on,
858 NULL);
859 if (inode)
860 inode_unref(inode);
861 if (replies)
862 afr_replies_wipe(replies, priv->child_count);
863 if (par_replies)
864 afr_replies_wipe(par_replies, priv->child_count);
865 if (xattr)
866 dict_unref(xattr);
867
868 return ret;
869 }
870
871 static inode_t *
872 afr_shd_entry_changes_index_inode(xlator_t *this, xlator_t *subvol,
873 uuid_t pargfid)
874 {
875 int ret = -1;
876 void *index_gfid = NULL;
877 loc_t rootloc = {
878 0,
879 };
880 loc_t loc = {
881 0,
882 };
883 dict_t *xattr = NULL;
884 inode_t *inode = NULL;
885 struct iatt iatt = {
886 0,
887 };
888
889 rootloc.inode = inode_ref(this->itable->root);
890 gf_uuid_copy(rootloc.gfid, rootloc.inode->gfid);
891
892 ret = syncop_getxattr(subvol, &rootloc, &xattr,
893 GF_XATTROP_ENTRY_CHANGES_GFID, NULL, NULL);
894 if (ret || !xattr) {
895 errno = -ret;
896 goto out;
897 }
898
899 ret = dict_get_ptr(xattr, GF_XATTROP_ENTRY_CHANGES_GFID, &index_gfid);
900 if (ret) {
901 errno = EINVAL;
902 goto out;
903 }
904
905 loc.inode = inode_new(this->itable);
906 if (!loc.inode) {
907 errno = ENOMEM;
908 goto out;
909 }
910
911 gf_uuid_copy(loc.pargfid, index_gfid);
912 loc.name = gf_strdup(uuid_utoa(pargfid));
913
914 ret = syncop_lookup(subvol, &loc, &iatt, NULL, NULL, NULL);
915 if (ret < 0) {
916 errno = -ret;
917 goto out;
918 }
919
920 inode = inode_link(loc.inode, NULL, NULL, &iatt);
921
922 out:
923 if (xattr)
924 dict_unref(xattr);
925 loc_wipe(&rootloc);
926 GF_FREE((char *)loc.name);
927 loc_wipe(&loc);
928
929 return inode;
930 }
931
932 static int
933 afr_selfheal_entry_do_subvol(call_frame_t *frame, xlator_t *this, fd_t *fd,
934 int child)
935 {
936 int ret = 0;
937 gf_dirent_t entries;
938 gf_dirent_t *entry = NULL;
939 off_t offset = 0;
940 call_frame_t *iter_frame = NULL;
941 xlator_t *subvol = NULL;
942 afr_private_t *priv = NULL;
943 gf_boolean_t mismatch = _gf_false;
944 afr_local_t *local = NULL;
945 loc_t loc = {
946 0,
947 };
948
949 priv = this->private;
950 subvol = priv->children[child];
951
952 INIT_LIST_HEAD(&entries.list);
953
954 local = frame->local;
955
956 iter_frame = afr_copy_frame(frame);
957 if (!iter_frame)
958 return -ENOMEM;
959
960 loc.inode = afr_shd_entry_changes_index_inode(this, subvol,
961 fd->inode->gfid);
962
963 while ((ret = syncop_readdir(subvol, fd, 131072, offset, &entries, NULL,
964 NULL))) {
965 if (ret > 0)
966 ret = 0;
967 list_for_each_entry(entry, &entries.list, list)
968 {
969 offset = entry->d_off;
970 /* skip . and .. */
971 if (inode_dir_or_parentdir(entry))
972 continue;
973
974 ret = afr_selfheal_entry_dirent(iter_frame, this, fd, entry->d_name,
975 loc.inode, subvol,
976 local->need_full_crawl);
977 AFR_STACK_RESET(iter_frame);
978 if (iter_frame->local == NULL) {
979 ret = -ENOTCONN;
980 break;
981 }
982
983 if (ret == -EIO) {
984 /* gfid or type mismatch. */
985 mismatch = _gf_true;
986 ret = 0;
987 }
988 if (ret)
989 break;
990 }
991
992 gf_dirent_free(&entries);
993 if (ret)
994 break;
995 }
996
997 loc_wipe(&loc);
998
999 AFR_STACK_DESTROY(iter_frame);
1000 if (mismatch == _gf_true)
1001 /* undo pending will be skipped */
1002 ret = -1;
1003 return ret;
1004 }
1005
1006 static int
1007 afr_selfheal_entry_granular_dirent(xlator_t *subvol, gf_dirent_t *entry,
1008 loc_t *parent, void *data)
1009 {
1010 int ret = 0;
1011 afr_granular_esh_args_t *args = data;
1012
1013 ret = afr_selfheal_entry_dirent(args->frame, args->xl, args->heal_fd,
1014 entry->d_name, parent->inode, subvol,
1015 _gf_false);
1016 AFR_STACK_RESET(args->frame);
1017 if (args->frame->local == NULL)
1018 ret = -ENOTCONN;
1019
1020 if (ret == -1)
1021 args->mismatch = _gf_true;
1022
1023 return ret;
1024 }
1025
1026 static int
1027 afr_selfheal_entry_granular(call_frame_t *frame, xlator_t *this, fd_t *fd,
1028 int subvol_idx, gf_boolean_t is_src)
1029 {
1030 int ret = 0;
1031 loc_t loc = {
1032 0,
1033 };
1034 xlator_t *subvol = NULL;
1035 afr_private_t *priv = NULL;
1036 afr_granular_esh_args_t args = {
1037 0,
1038 };
1039
1040 priv = this->private;
1041 subvol = priv->children[subvol_idx];
1042
1043 args.frame = afr_copy_frame(frame);
1044 if (!args.frame)
1045 goto out;
1046 args.xl = this;
1047 /* args.heal_fd represents the fd associated with the original directory
1048 * on which entry heal is being attempted.
1049 */
1050 args.heal_fd = fd;
1051
1052 /* @subvol here represents the subvolume of AFR where
1053 * indices/entry-changes/<pargfid> will be processed
1054 */
1055 loc.inode = afr_shd_entry_changes_index_inode(this, subvol,
1056 fd->inode->gfid);
1057 if (!loc.inode) {
1058 /* If granular heal failed on the sink (as it might sometimes
1059 * because it is the src that would mostly contain the granular
1060 * changelogs and the sink's entry-changes would be empty),
1061 * do not treat heal as failure.
1062 */
1063 if (is_src)
1064 ret = -errno;
1065 else
1066 ret = 0;
1067 goto out;
1068 }
1069
1070 ret = syncop_dir_scan(subvol, &loc, GF_CLIENT_PID_SELF_HEALD, &args,
1071 afr_selfheal_entry_granular_dirent);
1072
1073 loc_wipe(&loc);
1074
1075 if (args.mismatch == _gf_true)
1076 ret = -1;
1077 out:
1078 if (args.frame)
1079 AFR_STACK_DESTROY(args.frame);
1080 return ret;
1081 }
1082
1083 static int
1084 afr_selfheal_entry_do(call_frame_t *frame, xlator_t *this, fd_t *fd, int source,
1085 unsigned char *sources, unsigned char *healed_sinks)
1086 {
1087 int i = 0;
1088 int ret = 0;
1089 gf_boolean_t mismatch = _gf_false;
1090 afr_local_t *local = NULL;
1091 afr_private_t *priv = NULL;
1092
1093 priv = this->private;
1094 local = frame->local;
1095
1096 gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
1097 "performing %s entry selfheal on %s",
1098 (local->need_full_crawl ? "full" : "granular"),
1099 uuid_utoa(fd->inode->gfid));
1100
1101 for (i = 0; i < priv->child_count; i++) {
1102 /* Expunge */
1103 if (!healed_sinks[i])
1104 continue;
1105
1106 if (!local->need_full_crawl)
1107 /* Why call afr_selfheal_entry_granular() on a "healed sink",
1108 * given that it is the source that contains the granular
1109 * indices?
1110 * If the index for this directory is non-existent or empty on
1111 * this subvol (=> clear sink), then it will return early
1112 * without failure status.
1113 * If the index is non-empty and it is yet a 'healed sink', then
1114 * it is due to a split-brain in which case we anyway need to
1115 * crawl the indices/entry-changes/pargfid directory.
1116 */
1117 ret = afr_selfheal_entry_granular(frame, this, fd, i, _gf_false);
1118 else
1119 ret = afr_selfheal_entry_do_subvol(frame, this, fd, i);
1120
1121 if (ret == -EIO) {
1122 /* gfid or type mismatch. */
1123 mismatch = _gf_true;
1124 ret = 0;
1125 }
1126 if (ret)
1127 break;
1128 }
1129
1130 if (!ret && source != -1) {
1131 /* Impunge */
1132 if (local->need_full_crawl)
1133 ret = afr_selfheal_entry_do_subvol(frame, this, fd, source);
1134 else
1135 ret = afr_selfheal_entry_granular(frame, this, fd, source,
1136 _gf_true);
1137 }
1138
1139 if (mismatch == _gf_true)
1140 /* undo pending will be skipped */
1141 ret = -1;
1142 return ret;
1143 }
1144
1145 static int
1146 __afr_selfheal_entry(call_frame_t *frame, xlator_t *this, fd_t *fd,
1147 unsigned char *locked_on)
1148 {
1149 int ret = -1;
1150 int source = -1;
1151 unsigned char *sources = NULL;
1152 unsigned char *sinks = NULL;
1153 unsigned char *data_lock = NULL;
1154 unsigned char *postop_lock = NULL;
1155 unsigned char *healed_sinks = NULL;
1156 unsigned char *undid_pending = NULL;
1157 struct afr_reply *locked_replies = NULL;
1158 afr_local_t *local = NULL;
1159 afr_private_t *priv = NULL;
1160 gf_boolean_t did_sh = _gf_true;
1161 char *heal_type = "granular entry";
1162
1163 priv = this->private;
1164 local = frame->local;
1165
1166 sources = alloca0(priv->child_count);
1167 sinks = alloca0(priv->child_count);
1168 healed_sinks = alloca0(priv->child_count);
1169 undid_pending = alloca0(priv->child_count);
1170 data_lock = alloca0(priv->child_count);
1171 postop_lock = alloca0(priv->child_count);
1172
1173 locked_replies = alloca0(sizeof(*locked_replies) * priv->child_count);
1174
1175 ret = afr_selfheal_entrylk(frame, this, fd->inode, this->name, NULL,
1176 data_lock);
1177 {
1178 if (ret < priv->child_count) {
1179 gf_msg_debug(this->name, 0,
1180 "%s: Skipping "
1181 "entry self-heal as only %d sub-volumes could "
1182 "be locked in %s domain",
1183 uuid_utoa(fd->inode->gfid), ret, this->name);
1184 ret = -ENOTCONN;
1185 goto unlock;
1186 }
1187
1188 ret = __afr_selfheal_entry_prepare(frame, this, fd->inode, data_lock,
1189 sources, sinks, healed_sinks,
1190 locked_replies, &source, NULL);
1191 if (AFR_COUNT(healed_sinks, priv->child_count) == 0) {
1192 did_sh = _gf_false;
1193 goto unlock;
1194 }
1195
1196 local->need_full_crawl = afr_need_full_heal(
1197 this, locked_replies, source, healed_sinks, AFR_ENTRY_TRANSACTION);
1198 }
1199 unlock:
1200 afr_selfheal_unentrylk(frame, this, fd->inode, this->name, NULL, data_lock,
1201 NULL);
1202 if (ret < 0)
1203 goto out;
1204
1205 if (!did_sh)
1206 goto out;
1207
1208 ret = afr_selfheal_entry_do(frame, this, fd, source, sources, healed_sinks);
1209 if (ret)
1210 goto out;
1211
1212 /* Take entrylks in xlator domain before doing post-op (undo-pending) in
1213 * entry self-heal. This is to prevent a parallel name self-heal on
1214 * an entry under @fd->inode from reading pending xattrs while it is
1215 * being modified by SHD after entry sh below, given that
1216 * name self-heal takes locks ONLY in xlator domain and is free to read
1217 * pending changelog in the absence of the following locking.
1218 */
1219 ret = afr_selfheal_entrylk(frame, this, fd->inode, this->name, NULL,
1220 postop_lock);
1221 {
1222 if (AFR_CMP(data_lock, postop_lock, priv->child_count) != 0) {
1223 gf_msg_debug(this->name, 0,
1224 "%s: Skipping "
1225 "post-op after entry self-heal as %d "
1226 "sub-volumes, as opposed to %d, "
1227 "could be locked in %s domain",
1228 uuid_utoa(fd->inode->gfid), ret,
1229 AFR_COUNT(data_lock, priv->child_count), this->name);
1230 ret = -ENOTCONN;
1231 goto postop_unlock;
1232 }
1233
1234 afr_selfheal_restore_time(frame, this, fd->inode, source, healed_sinks,
1235 locked_replies);
1236 ret = afr_selfheal_undo_pending(
1237 frame, this, fd->inode, sources, sinks, healed_sinks, undid_pending,
1238 AFR_ENTRY_TRANSACTION, locked_replies, postop_lock);
1239 }
1240 postop_unlock:
1241 afr_selfheal_unentrylk(frame, this, fd->inode, this->name, NULL,
1242 postop_lock, NULL);
1243 out:
1244 if (did_sh) {
1245 if (local->need_full_crawl) {
1246 heal_type = "full entry";
1247 }
1248 afr_log_selfheal(fd->inode->gfid, this, ret, heal_type, source, sources,
1249 healed_sinks);
1250 } else {
1251 ret = 1;
1252 }
1253
1254 if (locked_replies)
1255 afr_replies_wipe(locked_replies, priv->child_count);
1256 return ret;
1257 }
1258
1259 static fd_t *
1260 afr_selfheal_data_opendir(xlator_t *this, inode_t *inode)
1261 {
1262 loc_t loc = {
1263 0,
1264 };
1265 int ret = 0;
1266 fd_t *fd = NULL;
1267
1268 fd = fd_create(inode, 0);
1269 if (!fd)
1270 return NULL;
1271
1272 loc.inode = inode_ref(inode);
1273 gf_uuid_copy(loc.gfid, inode->gfid);
1274
1275 ret = syncop_opendir(this, &loc, fd, NULL, NULL);
1276 if (ret) {
1277 fd_unref(fd);
1278 fd = NULL;
1279 } else {
1280 fd_bind(fd);
1281 }
1282
1283 loc_wipe(&loc);
1284 return fd;
1285 }
1286
1287 int
1288 afr_selfheal_entry(call_frame_t *frame, xlator_t *this, inode_t *inode)
1289 {
1290 afr_private_t *priv = NULL;
1291 unsigned char *locked_on = NULL;
1292 fd_t *fd = NULL;
1293 int ret = 0;
1294
1295 priv = this->private;
1296
1297 fd = afr_selfheal_data_opendir(this, inode);
1298 if (!fd)
1299 return -EIO;
1300
1301 locked_on = alloca0(priv->child_count);
1302
1303 ret = afr_selfheal_tie_breaker_entrylk(frame, this, inode, priv->sh_domain,
1304 NULL, locked_on);
1305 {
1306 if (ret < priv->child_count) {
1307 gf_msg_debug(this->name, 0,
1308 "%s: Skipping "
1309 "entry self-heal as only %d sub-volumes could "
1310 "be locked in %s domain",
1311 uuid_utoa(fd->inode->gfid), ret, priv->sh_domain);
1312 /* Either less than two subvols available, or another
1313 selfheal (from another server) is in progress. Skip
1314 for now in any case there isn't anything to do.
1315 */
1316 ret = -ENOTCONN;
1317 goto unlock;
1318 }
1319
1320 ret = __afr_selfheal_entry(frame, this, fd, locked_on);
1321 }
1322 unlock:
1323 afr_selfheal_unentrylk(frame, this, inode, priv->sh_domain, NULL, locked_on,
1324 NULL);
1325
1326 if (fd)
1327 fd_unref(fd);
1328
1329 return ret;
1330 }