Debian Codegraph @

xlators/cluster/afr/src/afr-common.c

raw
1 /*
2 Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
3 This file is part of GlusterFS.
4
5 This file is licensed to you under your choice of the GNU Lesser
6 General Public License, version 3 or any later version (LGPLv3 or
7 later), or the GNU General Public License, version 2 (GPLv2), in all
8 cases as published by the Free Software Foundation.
9 */
10
11 #include <libgen.h>
12 #include <unistd.h>
13 #include <fnmatch.h>
14 #include <sys/time.h>
15 #include <stdlib.h>
16 #include <signal.h>
17
18 #include "afr.h"
19 #include <glusterfs/dict.h>
20 #include <glusterfs/hashfn.h>
21 #include <glusterfs/list.h>
22 #include <glusterfs/compat-errno.h>
23 #include <glusterfs/compat.h>
24 #include <glusterfs/statedump.h>
25 #include <glusterfs/events.h>
26 #include <glusterfs/upcall-utils.h>
27
28 #include "afr-inode-read.h"
29 #include "afr-inode-write.h"
30 #include "afr-dir-read.h"
31 #include "afr-dir-write.h"
32 #include "afr-transaction.h"
33 #include "afr-self-heal.h"
34 #include "afr-self-heald.h"
35 #include "afr-messages.h"
36
37 static afr_fd_ctx_t *
38 __afr_fd_ctx_get(fd_t *fd, xlator_t *this);
39
40 static void
41 afr_set_need_heal(xlator_t *this, afr_local_t *local);
42
43 static int
44 __afr_inode_need_refresh_set(inode_t *inode, xlator_t *this);
45
46 int32_t
47 afr_quorum_errno(afr_private_t *priv)
48 {
49 return ENOTCONN;
50 }
51
52 gf_boolean_t
53 afr_is_private_directory(afr_private_t *priv, const char *name, pid_t pid)
54 {
55 if (strcmp(name, GF_REPLICATE_TRASH_DIR) == 0) {
56 /*For backward compatibility /.landfill is private*/
57 return _gf_true;
58 }
59
60 if (pid == GF_CLIENT_PID_GSYNCD) {
61 /*geo-rep needs to create/sync private directory on secondary because
62 * it appears in changelog*/
63 return _gf_false;
64 }
65
66 if (pid == GF_CLIENT_PID_GLFS_HEAL || pid == GF_CLIENT_PID_SELF_HEALD) {
67 if (strcmp(name, priv->anon_inode_name) == 0) {
68 /* anonymous-inode dir is private*/
69 return _gf_true;
70 }
71 } else {
72 if (strncmp(name, AFR_ANON_DIR_PREFIX, strlen(AFR_ANON_DIR_PREFIX)) ==
73 0) {
74 /* anonymous-inode dir prefix is private for geo-rep to work*/
75 return _gf_true;
76 }
77 }
78
79 return _gf_false;
80 }
81
82 void
83 afr_fill_success_replies(afr_local_t *local, afr_private_t *priv,
84 unsigned char *replies)
85 {
86 int i = 0;
87
88 for (i = 0; i < priv->child_count; i++) {
89 if (local->replies[i].valid && local->replies[i].op_ret == 0) {
90 replies[i] = 1;
91 } else {
92 replies[i] = 0;
93 }
94 }
95 }
96
97 int
98 afr_fav_child_reset_sink_xattrs(void *opaque);
99
100 static int
101 afr_fav_child_reset_sink_xattrs_cbk(int ret, call_frame_t *frame, void *opaque);
102
103 static void
104 afr_discover_done(call_frame_t *frame, xlator_t *this);
105
106 static int
107 afr_dom_lock_acquire_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
108 int op_ret, int op_errno, dict_t *xdata)
109 {
110 afr_local_t *local = frame->local;
111 afr_private_t *priv = this->private;
112 int i = (long)cookie;
113
114 local->cont.lk.dom_lock_op_ret[i] = op_ret;
115 local->cont.lk.dom_lock_op_errno[i] = op_errno;
116 if (op_ret < 0) {
117 gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
118 "%s: Failed to acquire %s on %s",
119 uuid_utoa(local->fd->inode->gfid), AFR_LK_HEAL_DOM,
120 priv->children[i]->name);
121 } else {
122 local->cont.lk.dom_locked_nodes[i] = 1;
123 }
124
125 syncbarrier_wake(&local->barrier);
126
127 return 0;
128 }
129
130 static int
131 afr_dom_lock_acquire(call_frame_t *frame)
132 {
133 afr_local_t *local = NULL;
134 afr_private_t *priv = NULL;
135 struct gf_flock flock = {
136 0,
137 };
138 int i = 0;
139
140 priv = frame->this->private;
141 local = frame->local;
142 local->cont.lk.dom_locked_nodes = GF_CALLOC(
143 priv->child_count, sizeof(*local->cont.lk.locked_nodes),
144 gf_afr_mt_char);
145 if (!local->cont.lk.dom_locked_nodes) {
146 return -ENOMEM;
147 }
148 local->cont.lk.dom_lock_op_ret = GF_CALLOC(
149 priv->child_count, sizeof(*local->cont.lk.dom_lock_op_ret),
150 gf_afr_mt_int32_t);
151 if (!local->cont.lk.dom_lock_op_ret) {
152 return -ENOMEM; /* CALLOC'd members are freed in afr_local_cleanup. */
153 }
154 local->cont.lk.dom_lock_op_errno = GF_CALLOC(
155 priv->child_count, sizeof(*local->cont.lk.dom_lock_op_errno),
156 gf_afr_mt_int32_t);
157 if (!local->cont.lk.dom_lock_op_errno) {
158 return -ENOMEM; /* CALLOC'd members are freed in afr_local_cleanup. */
159 }
160 flock.l_type = F_WRLCK;
161
162 AFR_ONALL(frame, afr_dom_lock_acquire_cbk, finodelk, AFR_LK_HEAL_DOM,
163 local->fd, F_SETLK, &flock, NULL);
164
165 if (!afr_has_quorum(local->cont.lk.dom_locked_nodes, priv, NULL))
166 goto blocking_lock;
167
168 /*If any of the bricks returned EAGAIN, we still need blocking locks.*/
169 if (AFR_COUNT(local->cont.lk.dom_locked_nodes, priv->child_count) !=
170 priv->child_count) {
171 for (i = 0; i < priv->child_count; i++) {
172 if (local->cont.lk.dom_lock_op_ret[i] == -1 &&
173 local->cont.lk.dom_lock_op_errno[i] == EAGAIN)
174 goto blocking_lock;
175 }
176 }
177
178 return 0;
179
180 blocking_lock:
181 afr_dom_lock_release(frame);
182 AFR_ONALL(frame, afr_dom_lock_acquire_cbk, finodelk, AFR_LK_HEAL_DOM,
183 local->fd, F_SETLKW, &flock, NULL);
184 if (!afr_has_quorum(local->cont.lk.dom_locked_nodes, priv, NULL)) {
185 afr_dom_lock_release(frame);
186 return -afr_quorum_errno(priv);
187 }
188
189 return 0;
190 }
191
192 static int
193 afr_dom_lock_release_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
194 int op_ret, int op_errno, dict_t *xdata)
195 {
196 afr_local_t *local = frame->local;
197 afr_private_t *priv = this->private;
198 int i = (long)cookie;
199
200 if (op_ret < 0) {
201 gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
202 "%s: Failed to release %s on %s", local->loc.path,
203 AFR_LK_HEAL_DOM, priv->children[i]->name);
204 }
205 local->cont.lk.dom_locked_nodes[i] = 0;
206
207 syncbarrier_wake(&local->barrier);
208
209 return 0;
210 }
211
212 void
213 afr_dom_lock_release(call_frame_t *frame)
214 {
215 afr_local_t *local = NULL;
216 afr_private_t *priv = NULL;
217 unsigned char *locked_on = NULL;
218 struct gf_flock flock = {
219 0,
220 };
221
222 local = frame->local;
223 priv = frame->this->private;
224 locked_on = local->cont.lk.dom_locked_nodes;
225 if (AFR_COUNT(locked_on, priv->child_count) == 0)
226 return;
227 flock.l_type = F_UNLCK;
228
229 AFR_ONLIST(locked_on, frame, afr_dom_lock_release_cbk, finodelk,
230 AFR_LK_HEAL_DOM, local->fd, F_SETLK, &flock, NULL);
231
232 return;
233 }
234
235 static void
236 afr_lk_heal_info_cleanup(afr_lk_heal_info_t *info)
237 {
238 if (!info)
239 return;
240 if (info->xdata_req)
241 dict_unref(info->xdata_req);
242 if (info->fd)
243 fd_unref(info->fd);
244 GF_FREE(info->locked_nodes);
245 GF_FREE(info->child_up_event_gen);
246 GF_FREE(info->child_down_event_gen);
247 GF_FREE(info);
248 }
249
250 static int
251 afr_add_lock_to_saved_locks(call_frame_t *frame, xlator_t *this)
252 {
253 afr_private_t *priv = this->private;
254 afr_local_t *local = frame->local;
255 afr_lk_heal_info_t *info = NULL;
256 afr_fd_ctx_t *fd_ctx = NULL;
257 int ret = -ENOMEM;
258
259 info = GF_CALLOC(sizeof(*info), 1, gf_afr_mt_lk_heal_info_t);
260 if (!info) {
261 goto cleanup;
262 }
263 INIT_LIST_HEAD(&info->pos);
264 info->fd = fd_ref(local->fd);
265 info->cmd = local->cont.lk.cmd;
266 info->pid = frame->root->pid;
267 gf_flock_copy(&info->flock, &local->cont.lk.user_flock);
268 info->xdata_req = dict_copy_with_ref(local->xdata_req, NULL);
269 if (!info->xdata_req) {
270 goto cleanup;
271 }
272 lk_owner_copy(&info->lk_owner, &frame->root->lk_owner);
273 info->locked_nodes = GF_MALLOC(
274 sizeof(*info->locked_nodes) * priv->child_count, gf_afr_mt_char);
275 if (!info->locked_nodes) {
276 goto cleanup;
277 }
278 memcpy(info->locked_nodes, local->cont.lk.locked_nodes,
279 sizeof(*info->locked_nodes) * priv->child_count);
280 info->child_up_event_gen = GF_CALLOC(sizeof(*info->child_up_event_gen),
281 priv->child_count, gf_afr_mt_int32_t);
282 if (!info->child_up_event_gen) {
283 goto cleanup;
284 }
285 info->child_down_event_gen = GF_CALLOC(sizeof(*info->child_down_event_gen),
286 priv->child_count,
287 gf_afr_mt_int32_t);
288 if (!info->child_down_event_gen) {
289 goto cleanup;
290 }
291
292 LOCK(&local->fd->lock);
293 {
294 fd_ctx = __afr_fd_ctx_get(local->fd, this);
295 if (fd_ctx)
296 fd_ctx->lk_heal_info = info;
297 }
298 UNLOCK(&local->fd->lock);
299 if (!fd_ctx) {
300 goto cleanup;
301 }
302
303 LOCK(&priv->lock);
304 {
305 list_add_tail(&info->pos, &priv->saved_locks);
306 }
307 UNLOCK(&priv->lock);
308
309 return 0;
310 cleanup:
311 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_LK_HEAL_DOM,
312 "%s: Failed to add lock to healq",
313 uuid_utoa(local->fd->inode->gfid));
314 if (info) {
315 afr_lk_heal_info_cleanup(info);
316 if (fd_ctx) {
317 LOCK(&local->fd->lock);
318 {
319 fd_ctx->lk_heal_info = NULL;
320 }
321 UNLOCK(&local->fd->lock);
322 }
323 }
324 return ret;
325 }
326
327 static int
328 afr_remove_lock_from_saved_locks(afr_local_t *local, xlator_t *this)
329 {
330 afr_private_t *priv = this->private;
331 struct gf_flock *user_flock;
332 afr_lk_heal_info_t *info = NULL;
333 afr_fd_ctx_t *fd_ctx = NULL;
334 int ret = -EINVAL;
335
336 fd_ctx = afr_fd_ctx_get(local->fd, this);
337 if (!fd_ctx || !fd_ctx->lk_heal_info) {
338 goto out;
339 }
340
341 user_flock = &local->cont.lk.user_flock;
342 info = fd_ctx->lk_heal_info;
343 if ((info->flock.l_start != user_flock->l_start) ||
344 (info->flock.l_whence != user_flock->l_whence) ||
345 (info->flock.l_len != user_flock->l_len)) {
346 /*TODO: Compare lkowners too.*/
347 goto out;
348 }
349
350 LOCK(&priv->lock);
351 {
352 list_del(&fd_ctx->lk_heal_info->pos);
353 }
354 UNLOCK(&priv->lock);
355
356 afr_lk_heal_info_cleanup(info);
357 fd_ctx->lk_heal_info = NULL;
358 return 0;
359 out:
360 if (ret)
361 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_LK_HEAL_DOM,
362 "%s: Failed to remove lock from healq",
363 uuid_utoa(local->fd->inode->gfid));
364 return ret;
365 }
366
367 static int
368 afr_lock_heal_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
369 int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
370 dict_t *xdata)
371 {
372 afr_local_t *local = frame->local;
373 int i = (long)cookie;
374
375 local->replies[i].valid = 1;
376 local->replies[i].op_ret = op_ret;
377 local->replies[i].op_errno = op_errno;
378 if (op_ret != 0) {
379 gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
380 "Failed to heal lock on child %d for %s", i,
381 uuid_utoa(local->fd->inode->gfid));
382 }
383 syncbarrier_wake(&local->barrier);
384 return 0;
385 }
386
387 static int
388 afr_getlk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
389 int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
390 {
391 afr_local_t *local = frame->local;
392 int i = (long)cookie;
393
394 local->replies[i].valid = 1;
395 local->replies[i].op_ret = op_ret;
396 local->replies[i].op_errno = op_errno;
397 if (op_ret != 0) {
398 gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
399 "Failed getlk for %s", uuid_utoa(local->fd->inode->gfid));
400 } else {
401 gf_flock_copy(&local->cont.lk.getlk_rsp[i], lock);
402 }
403
404 syncbarrier_wake(&local->barrier);
405 return 0;
406 }
407
408 static gf_boolean_t
409 afr_does_lk_owner_match(call_frame_t *frame, afr_private_t *priv,
410 afr_lk_heal_info_t *info)
411 {
412 int i = 0;
413 afr_local_t *local = frame->local;
414 struct gf_flock flock;
415 gf_boolean_t ret = _gf_true;
416 char *wind_on = alloca0(priv->child_count);
417 unsigned char *success_replies = alloca0(priv->child_count);
418 local->cont.lk.getlk_rsp = GF_CALLOC(sizeof(*local->cont.lk.getlk_rsp),
419 priv->child_count, gf_afr_mt_gf_lock);
420
421 gf_flock_copy(&flock, &info->flock);
422 for (i = 0; i < priv->child_count; i++) {
423 if (info->locked_nodes[i])
424 wind_on[i] = 1;
425 }
426
427 AFR_ONLIST(wind_on, frame, afr_getlk_cbk, lk, info->fd, F_GETLK, &flock,
428 info->xdata_req);
429
430 afr_fill_success_replies(local, priv, success_replies);
431 if (AFR_COUNT(success_replies, priv->child_count) == 0) {
432 ret = _gf_false;
433 goto out;
434 }
435
436 for (i = 0; i < priv->child_count; i++) {
437 if (!local->replies[i].valid || local->replies[i].op_ret != 0)
438 continue;
439 if (local->cont.lk.getlk_rsp[i].l_type == F_UNLCK)
440 continue;
441 /*TODO: Do we really need to compare lkowner if F_UNLCK is true?*/
442 if (!is_same_lkowner(&local->cont.lk.getlk_rsp[i].l_owner,
443 &info->lk_owner)) {
444 ret = _gf_false;
445 break;
446 }
447 }
448 out:
449 afr_local_replies_wipe(local, priv);
450 GF_FREE(local->cont.lk.getlk_rsp);
451 local->cont.lk.getlk_rsp = NULL;
452 return ret;
453 }
454
455 static void
456 afr_mark_fd_bad(fd_t *fd, xlator_t *this)
457 {
458 afr_fd_ctx_t *fd_ctx = NULL;
459
460 if (!fd)
461 return;
462 LOCK(&fd->lock);
463 {
464 fd_ctx = __afr_fd_ctx_get(fd, this);
465 if (fd_ctx) {
466 fd_ctx->is_fd_bad = _gf_true;
467 fd_ctx->lk_heal_info = NULL;
468 }
469 }
470 UNLOCK(&fd->lock);
471 }
472
473 static void
474 afr_add_lock_to_lkhealq(afr_private_t *priv, afr_lk_heal_info_t *info)
475 {
476 LOCK(&priv->lock);
477 {
478 list_del(&info->pos);
479 list_add_tail(&info->pos, &priv->lk_healq);
480 }
481 UNLOCK(&priv->lock);
482 }
483
484 static void
485 afr_lock_heal_do(call_frame_t *frame, afr_private_t *priv,
486 afr_lk_heal_info_t *info)
487 {
488 int i = 0;
489 int op_errno = 0;
490 int32_t *current_event_gen = NULL;
491 afr_local_t *local = frame->local;
492 xlator_t *this = frame->this;
493 char *wind_on = alloca0(priv->child_count);
494 gf_boolean_t retry = _gf_true;
495
496 frame->root->pid = info->pid;
497 lk_owner_copy(&frame->root->lk_owner, &info->lk_owner);
498
499 op_errno = -afr_dom_lock_acquire(frame);
500 if ((op_errno != 0)) {
501 goto release;
502 }
503
504 if (!afr_does_lk_owner_match(frame, priv, info)) {
505 gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_LK_HEAL_DOM,
506 "Ignoring lock heal for %s since lk-onwers mismatch. "
507 "Lock possibly pre-empted by another client.",
508 uuid_utoa(info->fd->inode->gfid));
509 goto release;
510 }
511
512 for (i = 0; i < priv->child_count; i++) {
513 if (info->locked_nodes[i])
514 continue;
515 wind_on[i] = 1;
516 }
517
518 current_event_gen = alloca(priv->child_count * sizeof(*current_event_gen));
519 memcpy(current_event_gen, info->child_up_event_gen,
520 priv->child_count * sizeof *current_event_gen);
521 AFR_ONLIST(wind_on, frame, afr_lock_heal_cbk, lk, info->fd, info->cmd,
522 &info->flock, info->xdata_req);
523
524 LOCK(&priv->lock);
525 {
526 for (i = 0; i < priv->child_count; i++) {
527 if (!wind_on[i])
528 continue;
529 if ((!local->replies[i].valid) || (local->replies[i].op_ret != 0)) {
530 continue;
531 }
532
533 if ((current_event_gen[i] == info->child_up_event_gen[i]) &&
534 (current_event_gen[i] > info->child_down_event_gen[i])) {
535 info->locked_nodes[i] = 1;
536 retry = _gf_false;
537 list_del_init(&info->pos);
538 list_add_tail(&info->pos, &priv->saved_locks);
539 } else {
540 /*We received subsequent child up/down events while heal was in
541 * progress; don't mark child as healed. Attempt again on the
542 * new child up*/
543 gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_LK_HEAL_DOM,
544 "Event gen mismatch: skipped healing lock on child %d "
545 "for %s.",
546 i, uuid_utoa(info->fd->inode->gfid));
547 }
548 }
549 }
550 UNLOCK(&priv->lock);
551
552 release:
553 afr_dom_lock_release(frame);
554 if (retry)
555 afr_add_lock_to_lkhealq(priv, info);
556 return;
557 }
558
559 static int
560 afr_lock_heal_done(int ret, call_frame_t *frame, void *opaque)
561 {
562 STACK_DESTROY(frame->root);
563 return 0;
564 }
565
566 static int
567 afr_lock_heal(void *opaque)
568 {
569 call_frame_t *frame = (call_frame_t *)opaque;
570 call_frame_t *iter_frame = NULL;
571 xlator_t *this = frame->this;
572 afr_private_t *priv = this->private;
573 afr_lk_heal_info_t *info = NULL;
574 afr_lk_heal_info_t *tmp = NULL;
575 struct list_head healq = {
576 0,
577 };
578 int ret = 0;
579
580 iter_frame = afr_copy_frame(frame);
581 if (!iter_frame) {
582 return ENOMEM;
583 }
584
585 INIT_LIST_HEAD(&healq);
586 LOCK(&priv->lock);
587 {
588 list_splice_init(&priv->lk_healq, &healq);
589 }
590 UNLOCK(&priv->lock);
591
592 list_for_each_entry_safe(info, tmp, &healq, pos)
593 {
594 GF_ASSERT((AFR_COUNT(info->locked_nodes, priv->child_count) <
595 priv->child_count));
596 ((afr_local_t *)(iter_frame->local))->fd = fd_ref(info->fd);
597 afr_lock_heal_do(iter_frame, priv, info);
598 AFR_STACK_RESET(iter_frame);
599 if (iter_frame->local == NULL) {
600 ret = ENOTCONN;
601 gf_msg(frame->this->name, GF_LOG_ERROR, ENOTCONN,
602 AFR_MSG_LK_HEAL_DOM,
603 "Aborting processing of lk_healq."
604 "Healing will be reattempted on next child up for locks "
605 "that are still in quorum.");
606 LOCK(&priv->lock);
607 {
608 list_add_tail(&healq, &priv->lk_healq);
609 }
610 UNLOCK(&priv->lock);
611 break;
612 }
613 }
614
615 AFR_STACK_DESTROY(iter_frame);
616 return ret;
617 }
618
619 static int
620 __afr_lock_heal_synctask(xlator_t *this, afr_private_t *priv, int child)
621 {
622 int ret = 0;
623 call_frame_t *frame = NULL;
624 afr_lk_heal_info_t *info = NULL;
625 afr_lk_heal_info_t *tmp = NULL;
626
627 if (priv->shd.iamshd)
628 return 0;
629
630 list_for_each_entry_safe(info, tmp, &priv->saved_locks, pos)
631 {
632 info->child_up_event_gen[child] = priv->event_generation;
633 list_del_init(&info->pos);
634 list_add_tail(&info->pos, &priv->lk_healq);
635 }
636
637 frame = create_frame(this, this->ctx->pool);
638 if (!frame)
639 return -1;
640
641 ret = synctask_new(this->ctx->env, afr_lock_heal, afr_lock_heal_done, frame,
642 frame);
643 if (ret)
644 gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_LK_HEAL_DOM,
645 "Failed to launch lock heal synctask");
646
647 return ret;
648 }
649
650 static int
651 __afr_mark_pending_lk_heal(xlator_t *this, afr_private_t *priv, int child)
652 {
653 afr_lk_heal_info_t *info = NULL;
654 afr_lk_heal_info_t *tmp = NULL;
655
656 if (priv->shd.iamshd)
657 return 0;
658 list_for_each_entry_safe(info, tmp, &priv->saved_locks, pos)
659 {
660 info->child_down_event_gen[child] = priv->event_generation;
661 if (info->locked_nodes[child] == 1)
662 info->locked_nodes[child] = 0;
663 if (!afr_has_quorum(info->locked_nodes, priv, NULL)) {
664 /* Since the lock was lost on quorum no. of nodes, we should
665 * not attempt to heal it anymore. Some other client could have
666 * acquired the lock, modified data and released it and this
667 * client wouldn't know about it if we heal it.*/
668 afr_mark_fd_bad(info->fd, this);
669 list_del(&info->pos);
670 afr_lk_heal_info_cleanup(info);
671 /* We're not winding an unlock on the node where the lock is still
672 * present because when fencing logic switches over to the new
673 * client (since we marked the fd bad), it should preempt any
674 * existing lock. */
675 }
676 }
677 return 0;
678 }
679
680 gf_boolean_t
681 afr_is_consistent_io_possible(afr_local_t *local, afr_private_t *priv,
682 int32_t *op_errno)
683 {
684 if (priv->consistent_io && local->call_count != priv->child_count) {
685 gf_msg(THIS->name, GF_LOG_INFO, 0, AFR_MSG_SUBVOLS_DOWN,
686 "All subvolumes are not up");
687 if (op_errno)
688 *op_errno = ENOTCONN;
689 return _gf_false;
690 }
691 return _gf_true;
692 }
693
694 gf_boolean_t
695 afr_is_lock_mode_mandatory(dict_t *xdata)
696 {
697 int ret = 0;
698 uint32_t lk_mode = GF_LK_ADVISORY;
699
700 ret = dict_get_uint32(xdata, GF_LOCK_MODE, &lk_mode);
701 if (!ret && lk_mode == GF_LK_MANDATORY)
702 return _gf_true;
703
704 return _gf_false;
705 }
706
707 call_frame_t *
708 afr_copy_frame(call_frame_t *base)
709 {
710 afr_local_t *local = NULL;
711 call_frame_t *frame = NULL;
712 int op_errno = 0;
713
714 frame = copy_frame(base);
715 if (!frame)
716 return NULL;
717 local = AFR_FRAME_INIT(frame, op_errno);
718 if (!local) {
719 AFR_STACK_DESTROY(frame);
720 return NULL;
721 }
722
723 return frame;
724 }
725
726 /* Check if an entry or inode could be undergoing a transaction. */
727 static gf_boolean_t
728 afr_is_possibly_under_txn(afr_transaction_type type, afr_local_t *local,
729 const unsigned int child_count)
730 {
731 int i = 0;
732 int tmp = 0;
733 GF_UNUSED char *key = NULL;
734 int keylen = 0;
735
736 if (type == AFR_ENTRY_TRANSACTION) {
737 key = GLUSTERFS_PARENT_ENTRYLK;
738 keylen = SLEN(GLUSTERFS_PARENT_ENTRYLK);
739 } else if (type == AFR_DATA_TRANSACTION) {
740 /*FIXME: Use GLUSTERFS_INODELK_DOM_COUNT etc. once
741 * pl_inodelk_xattr_fill supports separate keys for different
742 * domains.*/
743 key = GLUSTERFS_INODELK_COUNT;
744 keylen = SLEN(GLUSTERFS_INODELK_COUNT);
745 }
746 for (i = 0; i < child_count; i++) {
747 if (!local->replies[i].xdata)
748 continue;
749 if (dict_get_int32n(local->replies[i].xdata, key, keylen, &tmp) == 0)
750 if (tmp)
751 return _gf_true;
752 }
753
754 return _gf_false;
755 }
756
757 static void
758 afr_inode_ctx_destroy(afr_inode_ctx_t *ctx)
759 {
760 int i = 0;
761
762 if (!ctx)
763 return;
764
765 for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++) {
766 GF_FREE(ctx->pre_op_done[i]);
767 }
768
769 GF_FREE(ctx);
770 }
771
772 afr_inode_ctx_t *
773 __afr_inode_ctx_get(xlator_t *this, inode_t *inode)
774 {
775 uint64_t ctx_int = 0;
776 int ret;
777 int i;
778 int num_locks;
779 afr_inode_ctx_t *ictx = NULL;
780 afr_lock_t *lock = NULL;
781 afr_private_t *priv = NULL;
782
783 ret = __inode_ctx_get(inode, this, &ctx_int);
784 if (ret == 0) {
785 ictx = (afr_inode_ctx_t *)(uintptr_t)ctx_int;
786 goto out;
787 }
788
789 ictx = GF_CALLOC(1, sizeof(afr_inode_ctx_t), gf_afr_mt_inode_ctx_t);
790 if (!ictx)
791 goto err;
792
793 priv = this->private;
794 for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++) {
795 ictx->pre_op_done[i] = GF_CALLOC(sizeof *ictx->pre_op_done[i],
796 priv->child_count, gf_afr_mt_int32_t);
797 if (!ictx->pre_op_done[i]) {
798 goto err;
799 }
800 }
801
802 num_locks = sizeof(ictx->lock) / sizeof(afr_lock_t);
803 for (i = 0; i < num_locks; i++) {
804 lock = &ictx->lock[i];
805 INIT_LIST_HEAD(&lock->post_op);
806 INIT_LIST_HEAD(&lock->frozen);
807 INIT_LIST_HEAD(&lock->waiting);
808 INIT_LIST_HEAD(&lock->owners);
809 }
810
811 ctx_int = (uint64_t)(uintptr_t)ictx;
812 ret = __inode_ctx_set(inode, this, &ctx_int);
813 if (ret) {
814 goto err;
815 }
816
817 ictx->spb_choice = -1;
818 ictx->read_subvol = 0;
819 ictx->write_subvol = 0;
820 ictx->lock_count = 0;
821 out:
822 return ictx;
823 err:
824 if (ictx) {
825 afr_inode_ctx_destroy(ictx);
826 }
827 return NULL;
828 }
829
830 /*
831 * INODE CTX 64-bit VALUE FORMAT FOR SMALL (<= 16) SUBVOL COUNTS:
832 *
833 * |<---------- 64bit ------------>|
834 * 63 32 31 16 15 0
835 * | EVENT_GEN | DATA | METADATA |
836 *
837 *
838 * METADATA (bit-0 .. bit-15): bitmap representing subvolumes from which
839 * metadata can be attempted to be read.
840 *
841 * bit-0 => priv->subvolumes[0]
842 * bit-1 => priv->subvolumes[1]
843 * ... etc. till bit-15
844 *
845 * DATA (bit-16 .. bit-31): bitmap representing subvolumes from which data
846 * can be attempted to be read.
847 *
848 * bit-16 => priv->subvolumes[0]
849 * bit-17 => priv->subvolumes[1]
850 * ... etc. till bit-31
851 *
852 * EVENT_GEN (bit-32 .. bit-63): event generation (i.e priv->event_generation)
853 * when DATA and METADATA was last updated.
854 *
855 * If EVENT_GEN is < priv->event_generation,
856 * or is 0, it means afr_inode_refresh() needs
857 * to be called to recalculate the bitmaps.
858 */
859
860 static int
861 __afr_set_in_flight_sb_status(xlator_t *this, afr_local_t *local,
862 inode_t *inode)
863 {
864 int i = 0;
865 int txn_type = 0;
866 int count = 0;
867 int index = -1;
868 uint16_t datamap_old = 0;
869 uint16_t metadatamap_old = 0;
870 uint16_t datamap = 0;
871 uint16_t metadatamap = 0;
872 uint16_t tmp_map = 0;
873 uint16_t mask = 0;
874 uint32_t event = 0;
875 uint64_t val = 0;
876 afr_private_t *priv = NULL;
877
878 priv = this->private;
879 txn_type = local->transaction.type;
880
881 if (txn_type == AFR_DATA_TRANSACTION)
882 val = local->inode_ctx->write_subvol;
883 else
884 val = local->inode_ctx->read_subvol;
885
886 metadatamap_old = metadatamap = (val & 0x000000000000ffff);
887 datamap_old = datamap = (val & 0x00000000ffff0000) >> 16;
888 event = (val & 0xffffffff00000000) >> 32;
889
890 if (txn_type == AFR_DATA_TRANSACTION)
891 tmp_map = datamap;
892 else if (txn_type == AFR_METADATA_TRANSACTION)
893 tmp_map = metadatamap;
894
895 count = gf_bits_count(tmp_map);
896
897 for (i = 0; i < priv->child_count; i++) {
898 if (!local->transaction.failed_subvols[i])
899 continue;
900
901 mask = 1 << i;
902 if (txn_type == AFR_METADATA_TRANSACTION)
903 metadatamap &= ~mask;
904 else if (txn_type == AFR_DATA_TRANSACTION)
905 datamap &= ~mask;
906 }
907
908 switch (txn_type) {
909 case AFR_METADATA_TRANSACTION:
910 if ((metadatamap_old != 0) && (metadatamap == 0) && (count == 1)) {
911 index = gf_bits_index(tmp_map);
912 local->transaction.in_flight_sb_errno = local->replies[index]
913 .op_errno;
914 local->transaction.in_flight_sb = _gf_true;
915 metadatamap |= (1 << index);
916 }
917 if (metadatamap_old != metadatamap) {
918 __afr_inode_need_refresh_set(inode, this);
919 }
920 break;
921
922 case AFR_DATA_TRANSACTION:
923 if ((datamap_old != 0) && (datamap == 0) && (count == 1)) {
924 index = gf_bits_index(tmp_map);
925 local->transaction.in_flight_sb_errno = local->replies[index]
926 .op_errno;
927 local->transaction.in_flight_sb = _gf_true;
928 datamap |= (1 << index);
929 }
930 if (datamap_old != datamap)
931 __afr_inode_need_refresh_set(inode, this);
932 break;
933
934 default:
935 break;
936 }
937
938 val = ((uint64_t)metadatamap) | (((uint64_t)datamap) << 16) |
939 (((uint64_t)event) << 32);
940
941 if (txn_type == AFR_DATA_TRANSACTION)
942 local->inode_ctx->write_subvol = val;
943 local->inode_ctx->read_subvol = val;
944
945 return 0;
946 }
947
948 gf_boolean_t
949 afr_is_symmetric_error(call_frame_t *frame, xlator_t *this)
950 {
951 afr_local_t *local = NULL;
952 afr_private_t *priv = NULL;
953 int op_errno = 0;
954 int i_errno = 0;
955 gf_boolean_t matching_errors = _gf_true;
956 int i = 0;
957
958 priv = this->private;
959 local = frame->local;
960
961 for (i = 0; i < priv->child_count; i++) {
962 if (!local->replies[i].valid)
963 continue;
964 if (local->replies[i].op_ret != -1) {
965 /* Operation succeeded on at least one subvol,
966 so it is not a failed-everywhere situation.
967 */
968 matching_errors = _gf_false;
969 break;
970 }
971 i_errno = local->replies[i].op_errno;
972
973 if (i_errno == ENOTCONN) {
974 /* ENOTCONN is not a symmetric error. We do not
975 know if the operation was performed on the
976 backend or not.
977 */
978 matching_errors = _gf_false;
979 break;
980 }
981
982 if (!op_errno) {
983 op_errno = i_errno;
984 } else if (op_errno != i_errno) {
985 /* Mismatching op_errno's */
986 matching_errors = _gf_false;
987 break;
988 }
989 }
990
991 return matching_errors;
992 }
993
994 int
995 afr_set_in_flight_sb_status(xlator_t *this, call_frame_t *frame, inode_t *inode)
996 {
997 int ret = -1;
998 afr_private_t *priv = NULL;
999 afr_local_t *local = NULL;
1000
1001 priv = this->private;
1002 local = frame->local;
1003
1004 /* If this transaction saw no failures, then exit. */
1005 if (AFR_COUNT(local->transaction.failed_subvols, priv->child_count) == 0)
1006 return 0;
1007
1008 if (afr_is_symmetric_error(frame, this))
1009 return 0;
1010
1011 LOCK(&inode->lock);
1012 {
1013 ret = __afr_set_in_flight_sb_status(this, local, inode);
1014 }
1015 UNLOCK(&inode->lock);
1016
1017 return ret;
1018 }
1019
1020 static int
1021 __afr_inode_read_subvol_get_small(inode_t *inode, xlator_t *this,
1022 unsigned char *data, unsigned char *metadata,
1023 int *event_p)
1024 {
1025 afr_private_t *priv = NULL;
1026 uint16_t datamap = 0;
1027 uint16_t metadatamap = 0;
1028 uint32_t event = 0;
1029 uint64_t val = 0;
1030 int i = 0;
1031 afr_inode_ctx_t *ctx = NULL;
1032
1033 priv = this->private;
1034
1035 ctx = __afr_inode_ctx_get(this, inode);
1036 if (ctx == NULL)
1037 return -1;
1038
1039 val = ctx->read_subvol;
1040
1041 metadatamap = (val & 0x000000000000ffff);
1042 datamap = (val & 0x00000000ffff0000) >> 16;
1043 event = (val & 0xffffffff00000000) >> 32;
1044
1045 for (i = 0; i < priv->child_count; i++) {
1046 if (metadata)
1047 metadata[i] = (metadatamap >> i) & 1;
1048 if (data)
1049 data[i] = (datamap >> i) & 1;
1050 }
1051
1052 if (event_p)
1053 *event_p = event;
1054 return 0;
1055 }
1056
1057 static int
1058 __afr_inode_read_subvol_set_small(inode_t *inode, xlator_t *this,
1059 unsigned char *data, unsigned char *metadata,
1060 int event)
1061 {
1062 afr_private_t *priv = NULL;
1063 uint16_t datamap = 0;
1064 uint16_t metadatamap = 0;
1065 uint64_t val = 0;
1066 int i = 0;
1067 afr_inode_ctx_t *ctx = NULL;
1068
1069 priv = this->private;
1070
1071 ctx = __afr_inode_ctx_get(this, inode);
1072 if (ctx == NULL)
1073 return -1;
1074
1075 for (i = 0; i < priv->child_count; i++) {
1076 if (data[i])
1077 datamap |= (1 << i);
1078 if (metadata[i])
1079 metadatamap |= (1 << i);
1080 }
1081
1082 val = ((uint64_t)metadatamap) | (((uint64_t)datamap) << 16) |
1083 (((uint64_t)event) << 32);
1084
1085 ctx->read_subvol = val;
1086
1087 return 0;
1088 }
1089
1090 static int
1091 __afr_inode_read_subvol_get(inode_t *inode, xlator_t *this, unsigned char *data,
1092 unsigned char *metadata, int *event_p)
1093 {
1094 afr_private_t *priv = NULL;
1095 int ret = -1;
1096
1097 priv = this->private;
1098
1099 if (priv->child_count <= 16)
1100 ret = __afr_inode_read_subvol_get_small(inode, this, data, metadata,
1101 event_p);
1102 else
1103 /* TBD: allocate structure with array and read from it */
1104 ret = -1;
1105
1106 return ret;
1107 }
1108
1109 static int
1110 __afr_inode_read_subvol_set(inode_t *inode, xlator_t *this, unsigned char *data,
1111 unsigned char *metadata, int event)
1112 {
1113 afr_private_t *priv = NULL;
1114 int ret = -1;
1115
1116 priv = this->private;
1117
1118 if (priv->child_count <= 16)
1119 ret = __afr_inode_read_subvol_set_small(inode, this, data, metadata,
1120 event);
1121 else
1122 ret = -1;
1123
1124 return ret;
1125 }
1126
1127 static int
1128 __afr_inode_split_brain_choice_set(inode_t *inode, xlator_t *this,
1129 int spb_choice)
1130 {
1131 afr_inode_ctx_t *ctx = NULL;
1132
1133 ctx = __afr_inode_ctx_get(this, inode);
1134 if (ctx == NULL)
1135 return -1;
1136
1137 ctx->spb_choice = spb_choice;
1138
1139 return 0;
1140 }
1141
1142 int
1143 afr_inode_read_subvol_get(inode_t *inode, xlator_t *this, unsigned char *data,
1144 unsigned char *metadata, int *event_p)
1145 {
1146 int ret = -1;
1147
1148 GF_VALIDATE_OR_GOTO(this->name, inode, out);
1149
1150 LOCK(&inode->lock);
1151 {
1152 ret = __afr_inode_read_subvol_get(inode, this, data, metadata, event_p);
1153 }
1154 UNLOCK(&inode->lock);
1155 out:
1156 return ret;
1157 }
1158
1159 int
1160 afr_inode_get_readable(call_frame_t *frame, inode_t *inode, xlator_t *this,
1161 unsigned char *readable, int *event_p, int type)
1162 {
1163 afr_private_t *priv = this->private;
1164 afr_local_t *local = frame->local;
1165 unsigned char *data = alloca0(priv->child_count);
1166 unsigned char *metadata = alloca0(priv->child_count);
1167 int data_count = 0;
1168 int metadata_count = 0;
1169 int event_generation = 0;
1170 int ret = 0;
1171
1172 ret = afr_inode_read_subvol_get(inode, this, data, metadata,
1173 &event_generation);
1174 if (ret == -1)
1175 return -EIO;
1176
1177 data_count = AFR_COUNT(data, priv->child_count);
1178 metadata_count = AFR_COUNT(metadata, priv->child_count);
1179
1180 if (inode->ia_type == IA_IFDIR) {
1181 /* For directories, allow even if it is in data split-brain. */
1182 if (type == AFR_METADATA_TRANSACTION || local->op == GF_FOP_STAT ||
1183 local->op == GF_FOP_FSTAT) {
1184 if (!metadata_count)
1185 return -EIO;
1186 }
1187 } else {
1188 /* For files, abort in case of data/metadata split-brain. */
1189 if (!data_count || !metadata_count) {
1190 return -EIO;
1191 }
1192 }
1193
1194 if (type == AFR_METADATA_TRANSACTION && readable)
1195 memcpy(readable, metadata, priv->child_count * sizeof *metadata);
1196 if (type == AFR_DATA_TRANSACTION && readable) {
1197 if (!data_count)
1198 memcpy(readable, local->child_up,
1199 priv->child_count * sizeof *readable);
1200 else
1201 memcpy(readable, data, priv->child_count * sizeof *data);
1202 }
1203 if (event_p)
1204 *event_p = event_generation;
1205 return 0;
1206 }
1207
1208 static int
1209 afr_inode_split_brain_choice_get(inode_t *inode, xlator_t *this,
1210 int *spb_choice)
1211 {
1212 int ret = 0;
1213 afr_inode_ctx_t *ctx = NULL;
1214
1215 LOCK(&inode->lock);
1216 {
1217 ctx = __afr_inode_ctx_get(this, inode);
1218 if (ctx) {
1219 *spb_choice = ctx->spb_choice;
1220 } else {
1221 ret = -1;
1222 }
1223 }
1224 UNLOCK(&inode->lock);
1225
1226 return ret;
1227 }
1228
1229 /*
1230 * frame is used to get the favourite policy. Since
1231 * afr_inode_split_brain_choice_get was called with afr_open, it is possible to
1232 * have a frame with out local->replies. So in that case, frame is passed as
1233 * null, hence this function will handle the frame NULL case.
1234 */
1235 int
1236 afr_split_brain_read_subvol_get(inode_t *inode, xlator_t *this,
1237 call_frame_t *frame, int *spb_subvol)
1238 {
1239 int ret = -1;
1240 afr_local_t *local = NULL;
1241 afr_private_t *priv = NULL;
1242
1243 GF_VALIDATE_OR_GOTO("afr", this, out);
1244 GF_VALIDATE_OR_GOTO(this->name, this->private, out);
1245 GF_VALIDATE_OR_GOTO(this->name, inode, out);
1246 GF_VALIDATE_OR_GOTO(this->name, spb_subvol, out);
1247
1248 priv = this->private;
1249
1250 ret = afr_inode_split_brain_choice_get(inode, this, spb_subvol);
1251 if (*spb_subvol < 0 && priv->fav_child_policy && frame && frame->local) {
1252 local = frame->local;
1253 *spb_subvol = afr_sh_get_fav_by_policy(this, local->replies, inode,
1254 NULL);
1255 if (*spb_subvol >= 0) {
1256 ret = 0;
1257 }
1258 }
1259
1260 out:
1261 return ret;
1262 }
1263
1264 static int
1265 afr_inode_read_subvol_set(inode_t *inode, xlator_t *this, unsigned char *data,
1266 unsigned char *metadata, int event)
1267 {
1268 int ret = -1;
1269
1270 GF_VALIDATE_OR_GOTO(this->name, inode, out);
1271
1272 LOCK(&inode->lock);
1273 {
1274 ret = __afr_inode_read_subvol_set(inode, this, data, metadata, event);
1275 }
1276 UNLOCK(&inode->lock);
1277 out:
1278 return ret;
1279 }
1280
1281 int
1282 afr_inode_split_brain_choice_set(inode_t *inode, xlator_t *this, int spb_choice)
1283 {
1284 int ret = -1;
1285
1286 GF_VALIDATE_OR_GOTO(this->name, inode, out);
1287
1288 LOCK(&inode->lock);
1289 {
1290 ret = __afr_inode_split_brain_choice_set(inode, this, spb_choice);
1291 }
1292 UNLOCK(&inode->lock);
1293 out:
1294 return ret;
1295 }
1296
1297 /* The caller of this should perform afr_inode_refresh, if this function
1298 * returns _gf_true
1299 */
1300 gf_boolean_t
1301 afr_is_inode_refresh_reqd(inode_t *inode, xlator_t *this, int event_gen1,
1302 int event_gen2)
1303 {
1304 gf_boolean_t need_refresh = _gf_false;
1305 afr_inode_ctx_t *ctx = NULL;
1306
1307 GF_VALIDATE_OR_GOTO(this->name, inode, out);
1308
1309 LOCK(&inode->lock);
1310 {
1311 ctx = __afr_inode_ctx_get(this, inode);
1312 if (ctx == NULL)
1313 goto unlock;
1314
1315 need_refresh = ctx->need_refresh;
1316 /* Hoping that the caller will do inode_refresh followed by
1317 * this, hence setting the need_refresh to false */
1318 ctx->need_refresh = _gf_false;
1319 }
1320 unlock:
1321 UNLOCK(&inode->lock);
1322
1323 if (event_gen1 != event_gen2)
1324 need_refresh = _gf_true;
1325 out:
1326 return need_refresh;
1327 }
1328
1329 static int
1330 __afr_inode_need_refresh_set(inode_t *inode, xlator_t *this)
1331 {
1332 afr_inode_ctx_t *ctx = NULL;
1333
1334 ctx = __afr_inode_ctx_get(this, inode);
1335 if (ctx) {
1336 ctx->need_refresh = _gf_true;
1337 return 0;
1338 }
1339
1340 return -1;
1341 }
1342
1343 int
1344 afr_inode_need_refresh_set(inode_t *inode, xlator_t *this)
1345 {
1346 int ret = -1;
1347
1348 GF_VALIDATE_OR_GOTO(this->name, inode, out);
1349
1350 LOCK(&inode->lock);
1351 {
1352 ret = __afr_inode_need_refresh_set(inode, this);
1353 }
1354 UNLOCK(&inode->lock);
1355 out:
1356 return ret;
1357 }
1358
1359 static void
1360 afr_spb_choice_timeout_cancel(xlator_t *this, inode_t *inode)
1361 {
1362 afr_inode_ctx_t *ctx = NULL;
1363
1364 if (!inode)
1365 return;
1366
1367 LOCK(&inode->lock);
1368 {
1369 ctx = __afr_inode_ctx_get(this, inode);
1370 if (ctx == NULL) {
1371 UNLOCK(&inode->lock);
1372 gf_msg(this->name, GF_LOG_WARNING, 0,
1373 AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR,
1374 "Failed to cancel split-brain choice timer.");
1375 return;
1376 }
1377 ctx->spb_choice = -1;
1378 if (ctx->timer) {
1379 gf_timer_call_cancel(this->ctx, ctx->timer);
1380 ctx->timer = NULL;
1381 }
1382 }
1383 UNLOCK(&inode->lock);
1384 }
1385
1386 static void
1387 afr_set_split_brain_choice_cbk(void *data)
1388 {
1389 inode_t *inode = data;
1390 xlator_t *this = THIS;
1391
1392 afr_spb_choice_timeout_cancel(this, inode);
1393 inode_invalidate(inode);
1394 inode_unref(inode);
1395 return;
1396 }
1397
1398 int
1399 afr_set_split_brain_choice(int ret, call_frame_t *frame, void *opaque)
1400 {
1401 int op_errno = ENOMEM;
1402 afr_private_t *priv = NULL;
1403 afr_inode_ctx_t *ctx = NULL;
1404 inode_t *inode = NULL;
1405 loc_t *loc = NULL;
1406 xlator_t *this = NULL;
1407 afr_spbc_timeout_t *data = opaque;
1408 struct timespec delta = {
1409 0,
1410 };
1411 gf_boolean_t timer_set = _gf_false;
1412 gf_boolean_t timer_cancelled = _gf_false;
1413 gf_boolean_t timer_reset = _gf_false;
1414 int old_spb_choice = -1;
1415
1416 frame = data->frame;
1417 loc = data->loc;
1418 this = frame->this;
1419 priv = this->private;
1420
1421 if (ret) {
1422 op_errno = -ret;
1423 ret = -1;
1424 goto out;
1425 }
1426
1427 delta.tv_sec = priv->spb_choice_timeout;
1428 delta.tv_nsec = 0;
1429
1430 if (!loc->inode) {
1431 ret = -1;
1432 op_errno = EINVAL;
1433 goto out;
1434 }
1435
1436 if (!(data->d_spb || data->m_spb)) {
1437 gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR,
1438 "Cannot set "
1439 "replica.split-brain-choice on %s. File is"
1440 " not in data/metadata split-brain.",
1441 uuid_utoa(loc->gfid));
1442 ret = -1;
1443 op_errno = EINVAL;
1444 goto out;
1445 }
1446
1447 /*
1448 * we're ref'ing the inode before LOCK like it is done elsewhere in the
1449 * code. If we ref after LOCK, coverity complains of possible deadlocks.
1450 */
1451 inode = inode_ref(loc->inode);
1452
1453 LOCK(&inode->lock);
1454 {
1455 ctx = __afr_inode_ctx_get(this, inode);
1456 if (ctx == NULL) {
1457 UNLOCK(&inode->lock);
1458 ret = -1;
1459 gf_msg(this->name, GF_LOG_ERROR, 0,
1460 AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR,
1461 "Failed to get inode_ctx for %s", loc->name);
1462 goto post_unlock;
1463 } else {
1464 ret = 0;
1465 }
1466
1467 old_spb_choice = ctx->spb_choice;
1468 ctx->spb_choice = data->spb_child_index;
1469
1470 /* Possible changes in spb-choice :
1471 * valid to -1 : cancel timer and unref
1472 * valid to valid : cancel timer and inject new one
1473 * -1 to -1 : unref and do not do anything
1474 * -1 to valid : inject timer
1475 */
1476
1477 /* ctx->timer is NULL iff previous value of
1478 * ctx->spb_choice is -1
1479 */
1480 if (ctx->timer) {
1481 if (ctx->spb_choice == -1) {
1482 if (!gf_timer_call_cancel(this->ctx, ctx->timer)) {
1483 ctx->timer = NULL;
1484 timer_cancelled = _gf_true;
1485 }
1486 /* If timer cancel failed here it means that the
1487 * previous cbk will be executed which will set
1488 * spb_choice to -1. So we can consider the
1489 * 'valid to -1' case to be a success
1490 * (i.e. ret = 0) and goto unlock.
1491 */
1492 goto unlock;
1493 }
1494 goto reset_timer;
1495 } else {
1496 if (ctx->spb_choice == -1)
1497 goto unlock;
1498 goto set_timer;
1499 }
1500
1501 reset_timer:
1502 ret = gf_timer_call_cancel(this->ctx, ctx->timer);
1503 if (ret != 0) {
1504 /* We need to bail out now instead of launching a new
1505 * timer. Otherwise the cbk of the previous timer event
1506 * will cancel the new ctx->timer.
1507 */
1508 ctx->spb_choice = old_spb_choice;
1509 ret = -1;
1510 op_errno = EAGAIN;
1511 goto unlock;
1512 }
1513 ctx->timer = NULL;
1514 timer_reset = _gf_true;
1515
1516 set_timer:
1517 ctx->timer = gf_timer_call_after(this->ctx, delta,
1518 afr_set_split_brain_choice_cbk, inode);
1519 if (!ctx->timer) {
1520 ctx->spb_choice = old_spb_choice;
1521 ret = -1;
1522 op_errno = ENOMEM;
1523 }
1524 if (!timer_reset && ctx->timer)
1525 timer_set = _gf_true;
1526 if (timer_reset && !ctx->timer)
1527 timer_cancelled = _gf_true;
1528 }
1529 unlock:
1530 UNLOCK(&inode->lock);
1531 post_unlock:
1532 if (!timer_set)
1533 inode_unref(inode);
1534 if (timer_cancelled)
1535 inode_unref(inode);
1536 /*
1537 * We need to invalidate the inode to prevent the kernel from serving
1538 * reads from an older cached value despite a change in spb_choice to
1539 * a new value.
1540 */
1541 inode_invalidate(inode);
1542 out:
1543 GF_FREE(data);
1544 AFR_STACK_UNWIND(setxattr, frame, ret, op_errno, NULL);
1545 return 0;
1546 }
1547
1548 static int
1549 afr_accused_fill(afr_private_t *priv, dict_t *xdata, unsigned char *accused,
1550 afr_transaction_type type)
1551 {
1552 int i = 0;
1553 int idx = afr_index_for_transaction_type(type);
1554 void *pending_raw = NULL;
1555 int ret = 0;
1556
1557 for (i = 0; i < priv->child_count; i++) {
1558 ret = dict_get_ptr(xdata, priv->pending_key[i], &pending_raw);
1559 if (ret) /* no pending flags */
1560 continue;
1561
1562 if (*((int *)pending_raw + idx))
1563 accused[i] = 1;
1564 }
1565
1566 return 0;
1567 }
1568
1569 static int
1570 afr_accuse_smallfiles(afr_private_t *priv, struct afr_reply *replies,
1571 unsigned char *data_accused)
1572 {
1573 int i = 0;
1574 uint64_t maxsize = 0;
1575
1576 for (i = 0; i < priv->child_count; i++) {
1577 if (replies[i].valid && replies[i].xdata &&
1578 dict_get_sizen(replies[i].xdata, GLUSTERFS_BAD_INODE))
1579 continue;
1580 if (data_accused[i])
1581 continue;
1582 if (replies[i].poststat.ia_size > maxsize)
1583 maxsize = replies[i].poststat.ia_size;
1584 }
1585
1586 for (i = 0; i < priv->child_count; i++) {
1587 if (data_accused[i])
1588 continue;
1589 if (AFR_IS_ARBITER_BRICK(priv, i))
1590 continue;
1591 if (replies[i].poststat.ia_size < maxsize)
1592 data_accused[i] = 1;
1593 }
1594
1595 return 0;
1596 }
1597
1598 static int
1599 afr_readables_fill(afr_local_t *local, xlator_t *this, inode_t *inode,
1600 unsigned char *data_accused, unsigned char *metadata_accused,
1601 unsigned char *data_readable,
1602 unsigned char *metadata_readable, struct afr_reply *replies)
1603 {
1604 afr_private_t *priv = NULL;
1605 dict_t *xdata = NULL;
1606 int i = 0;
1607 int ret = 0;
1608 ia_type_t ia_type = IA_INVAL;
1609
1610 priv = this->private;
1611
1612 for (i = 0; i < priv->child_count; i++) {
1613 data_readable[i] = 1;
1614 metadata_readable[i] = 1;
1615 }
1616 if (AFR_IS_ARBITER_BRICK(priv, ARBITER_BRICK_INDEX)) {
1617 data_readable[ARBITER_BRICK_INDEX] = 0;
1618 metadata_readable[ARBITER_BRICK_INDEX] = 0;
1619 }
1620
1621 for (i = 0; i < priv->child_count; i++) {
1622 if (replies) { /* Lookup */
1623 if (!replies[i].valid || replies[i].op_ret == -1 ||
1624 (replies[i].xdata &&
1625 dict_get_sizen(replies[i].xdata, GLUSTERFS_BAD_INODE))) {
1626 data_readable[i] = 0;
1627 metadata_readable[i] = 0;
1628 continue;
1629 }
1630
1631 xdata = replies[i].xdata;
1632 ia_type = replies[i].poststat.ia_type;
1633 } else { /* pre-op xattrop */
1634 xdata = local->transaction.changelog_xdata[i];
1635 ia_type = inode->ia_type;
1636 }
1637
1638 if (!xdata)
1639 continue; /* mkdir_cbk sends NULL xdata_rsp. */
1640 afr_accused_fill(priv, xdata, data_accused,
1641 (ia_type == IA_IFDIR) ? AFR_ENTRY_TRANSACTION
1642 : AFR_DATA_TRANSACTION);
1643
1644 afr_accused_fill(priv, xdata, metadata_accused,
1645 AFR_METADATA_TRANSACTION);
1646 }
1647
1648 if (replies && ia_type != IA_INVAL && ia_type != IA_IFDIR &&
1649 /* We want to accuse small files only when we know for
1650 * sure that there is no IO happening. Otherwise, the
1651 * ia_sizes obtained in post-refresh replies may
1652 * mismatch due to a race between inode-refresh and
1653 * ongoing writes, causing spurious heal launches*/
1654 !afr_is_possibly_under_txn(AFR_DATA_TRANSACTION, local,
1655 priv->child_count)) {
1656 afr_accuse_smallfiles(priv, replies, data_accused);
1657 }
1658
1659 for (i = 0; i < priv->child_count; i++) {
1660 if (data_accused[i]) {
1661 data_readable[i] = 0;
1662 ret = 1;
1663 }
1664 if (metadata_accused[i]) {
1665 metadata_readable[i] = 0;
1666 ret = 1;
1667 }
1668 }
1669 return ret;
1670 }
1671
1672 int
1673 afr_replies_interpret(afr_local_t *local, xlator_t *this, inode_t *inode,
1674 gf_boolean_t *start_heal)
1675 {
1676 afr_private_t *priv = NULL;
1677 struct afr_reply *replies = NULL;
1678 int event_generation = 0;
1679 int i = 0;
1680 unsigned char *data_accused = NULL;
1681 unsigned char *metadata_accused = NULL;
1682 unsigned char *data_readable = NULL;
1683 unsigned char *metadata_readable = NULL;
1684 int ret = 0;
1685
1686 priv = this->private;
1687 replies = local->replies;
1688 event_generation = local->event_generation;
1689
1690 data_accused = alloca0(priv->child_count);
1691 data_readable = alloca0(priv->child_count);
1692 metadata_accused = alloca0(priv->child_count);
1693 metadata_readable = alloca0(priv->child_count);
1694
1695 ret = afr_readables_fill(local, this, inode, data_accused, metadata_accused,
1696 data_readable, metadata_readable, replies);
1697
1698 for (i = 0; i < priv->child_count; i++) {
1699 if (start_heal && priv->child_up[i] &&
1700 (data_accused[i] || metadata_accused[i])) {
1701 *start_heal = _gf_true;
1702 break;
1703 }
1704 }
1705 afr_inode_read_subvol_set(inode, this, data_readable, metadata_readable,
1706 event_generation);
1707 return ret;
1708 }
1709
1710 static int
1711 afr_refresh_selfheal_done(int ret, call_frame_t *heal, void *opaque)
1712 {
1713 if (heal)
1714 AFR_STACK_DESTROY(heal);
1715 return 0;
1716 }
1717
1718 static int
1719 afr_inode_refresh_err(call_frame_t *frame, xlator_t *this)
1720 {
1721 afr_local_t *local = NULL;
1722 afr_private_t *priv = NULL;
1723 int i = 0;
1724 int err = 0;
1725
1726 local = frame->local;
1727 priv = this->private;
1728
1729 for (i = 0; i < priv->child_count; i++) {
1730 if (local->replies[i].valid && !local->replies[i].op_ret) {
1731 err = 0;
1732 goto ret;
1733 }
1734 }
1735
1736 err = afr_final_errno(local, priv);
1737 ret:
1738 return err;
1739 }
1740
1741 static gf_boolean_t
1742 afr_selfheal_enabled(const xlator_t *this)
1743 {
1744 const afr_private_t *priv = this->private;
1745
1746 return priv->data_self_heal || priv->metadata_self_heal ||
1747 priv->entry_self_heal;
1748 }
1749
1750 static int
1751 afr_txn_refresh_done(call_frame_t *frame, xlator_t *this, int err)
1752 {
1753 call_frame_t *heal_frame = NULL;
1754 afr_local_t *heal_local = NULL;
1755 afr_local_t *local = NULL;
1756 afr_private_t *priv = NULL;
1757 inode_t *inode = NULL;
1758 int event_generation = 0;
1759 int read_subvol = -1;
1760 int ret = 0;
1761
1762 local = frame->local;
1763 inode = local->inode;
1764 priv = this->private;
1765
1766 if (err)
1767 goto refresh_done;
1768
1769 if (local->op == GF_FOP_LOOKUP)
1770 goto refresh_done;
1771
1772 ret = afr_inode_get_readable(frame, inode, this, local->readable,
1773 &event_generation, local->transaction.type);
1774
1775 if (ret == -EIO) {
1776 /* No readable subvolume even after refresh ==> splitbrain.*/
1777 if (!priv->fav_child_policy) {
1778 err = EIO;
1779 goto refresh_done;
1780 }
1781 read_subvol = afr_sh_get_fav_by_policy(this, local->replies, inode,
1782 NULL);
1783 if (read_subvol == -1) {
1784 err = EIO;
1785 goto refresh_done;
1786 }
1787
1788 heal_frame = afr_frame_create(this, NULL);
1789 if (!heal_frame) {
1790 err = EIO;
1791 goto refresh_done;
1792 }
1793 heal_local = heal_frame->local;
1794 heal_local->xdata_req = dict_new();
1795 if (!heal_local->xdata_req) {
1796 err = EIO;
1797 AFR_STACK_DESTROY(heal_frame);
1798 goto refresh_done;
1799 }
1800 heal_local->heal_frame = frame;
1801 ret = synctask_new(this->ctx->env, afr_fav_child_reset_sink_xattrs,
1802 afr_fav_child_reset_sink_xattrs_cbk, heal_frame,
1803 heal_frame);
1804 return 0;
1805 }
1806
1807 refresh_done:
1808 afr_local_replies_wipe(local, this->private);
1809 local->refreshfn(frame, this, err);
1810
1811 return 0;
1812 }
1813
1814 static int
1815 afr_inode_refresh_done(call_frame_t *frame, xlator_t *this, int error)
1816 {
1817 call_frame_t *heal_frame = NULL;
1818 afr_local_t *local = NULL;
1819 afr_private_t *priv = NULL;
1820 gf_boolean_t start_heal = _gf_false;
1821 afr_local_t *heal_local = NULL;
1822 unsigned char *success_replies = NULL;
1823 int ret = 0;
1824
1825 if (error != 0) {
1826 goto refresh_done;
1827 }
1828
1829 local = frame->local;
1830 priv = this->private;
1831 success_replies = alloca0(priv->child_count);
1832 afr_fill_success_replies(local, priv, success_replies);
1833
1834 if (priv->thin_arbiter_count && local->is_read_txn &&
1835 AFR_COUNT(success_replies, priv->child_count) != priv->child_count) {
1836 /* We need to query the good bricks and/or thin-arbiter.*/
1837 if (success_replies[0]) {
1838 local->read_txn_query_child = AFR_CHILD_ZERO;
1839 } else if (success_replies[1]) {
1840 local->read_txn_query_child = AFR_CHILD_ONE;
1841 }
1842 error = EINVAL;
1843 goto refresh_done;
1844 }
1845
1846 if (!afr_has_quorum(success_replies, priv, frame)) {
1847 error = afr_final_errno(frame->local, this->private);
1848 if (!error)
1849 error = afr_quorum_errno(priv);
1850 goto refresh_done;
1851 }
1852
1853 ret = afr_replies_interpret(local, this, local->refreshinode, &start_heal);
1854
1855 if (ret && afr_selfheal_enabled(this) && start_heal) {
1856 heal_frame = afr_frame_create(this, NULL);
1857 if (!heal_frame)
1858 goto refresh_done;
1859 heal_local = heal_frame->local;
1860 heal_local->refreshinode = inode_ref(local->refreshinode);
1861 heal_local->heal_frame = heal_frame;
1862 if (!afr_throttled_selfheal(heal_frame, this)) {
1863 AFR_STACK_DESTROY(heal_frame);
1864 goto refresh_done;
1865 }
1866 }
1867
1868 refresh_done:
1869 afr_txn_refresh_done(frame, this, error);
1870
1871 return 0;
1872 }
1873
1874 static void
1875 afr_inode_refresh_subvol_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
1876 int op_ret, int op_errno, struct iatt *buf,
1877 dict_t *xdata, struct iatt *par)
1878 {
1879 afr_local_t *local = NULL;
1880 int call_child = (long)cookie;
1881 int8_t need_heal = 1;
1882 int call_count = 0;
1883 int ret = 0;
1884
1885 local = frame->local;
1886 local->replies[call_child].valid = 1;
1887 local->replies[call_child].op_ret = op_ret;
1888 local->replies[call_child].op_errno = op_errno;
1889 if (op_ret != -1) {
1890 local->replies[call_child].poststat = *buf;
1891 if (par)
1892 local->replies[call_child].postparent = *par;
1893 if (xdata)
1894 local->replies[call_child].xdata = dict_ref(xdata);
1895 }
1896
1897 if (xdata) {
1898 ret = dict_get_int8(xdata, "link-count", &need_heal);
1899 if (ret) {
1900 gf_msg_debug(this->name, -ret, "Unable to get link count");
1901 }
1902 }
1903
1904 local->replies[call_child].need_heal = need_heal;
1905 call_count = afr_frame_return(frame);
1906 if (call_count == 0) {
1907 afr_set_need_heal(this, local);
1908 ret = afr_inode_refresh_err(frame, this);
1909 if (ret) {
1910 gf_msg_debug(this->name, ret, "afr_inode_refresh_err failed");
1911 }
1912 afr_inode_refresh_done(frame, this, ret);
1913 }
1914 }
1915
1916 static int
1917 afr_inode_refresh_subvol_with_lookup_cbk(call_frame_t *frame, void *cookie,
1918 xlator_t *this, int op_ret,
1919 int op_errno, inode_t *inode,
1920 struct iatt *buf, dict_t *xdata,
1921 struct iatt *par)
1922 {
1923 afr_inode_refresh_subvol_cbk(frame, cookie, this, op_ret, op_errno, buf,
1924 xdata, par);
1925 return 0;
1926 }
1927
1928 static int
1929 afr_inode_refresh_subvol_with_lookup(call_frame_t *frame, xlator_t *this, int i,
1930 inode_t *inode, uuid_t gfid, dict_t *xdata)
1931 {
1932 loc_t loc = {
1933 0,
1934 };
1935 afr_private_t *priv = NULL;
1936
1937 priv = this->private;
1938
1939 loc.inode = inode;
1940 if (gf_uuid_is_null(inode->gfid) && gfid) {
1941 /* To handle setattr/setxattr on yet to be linked inode from
1942 * dht */
1943 gf_uuid_copy(loc.gfid, gfid);
1944 } else {
1945 gf_uuid_copy(loc.gfid, inode->gfid);
1946 }
1947
1948 STACK_WIND_COOKIE(frame, afr_inode_refresh_subvol_with_lookup_cbk,
1949 (void *)(long)i, priv->children[i],
1950 priv->children[i]->fops->lookup, &loc, xdata);
1951 return 0;
1952 }
1953
1954 int
1955 afr_inode_refresh_subvol_with_fstat_cbk(call_frame_t *frame, void *cookie,
1956 xlator_t *this, int32_t op_ret,
1957 int32_t op_errno, struct iatt *buf,
1958 dict_t *xdata)
1959 {
1960 afr_inode_refresh_subvol_cbk(frame, cookie, this, op_ret, op_errno, buf,
1961 xdata, NULL);
1962 return 0;
1963 }
1964
1965 static int
1966 afr_inode_refresh_subvol_with_fstat(call_frame_t *frame, xlator_t *this, int i,
1967 dict_t *xdata)
1968 {
1969 afr_private_t *priv = NULL;
1970 afr_local_t *local = NULL;
1971
1972 priv = this->private;
1973 local = frame->local;
1974
1975 STACK_WIND_COOKIE(frame, afr_inode_refresh_subvol_with_fstat_cbk,
1976 (void *)(long)i, priv->children[i],
1977 priv->children[i]->fops->fstat, local->fd, xdata);
1978 return 0;
1979 }
1980
1981 static int
1982 afr_inode_refresh_do(call_frame_t *frame, xlator_t *this)
1983 {
1984 afr_local_t *local = NULL;
1985 afr_private_t *priv = NULL;
1986 int call_count = 0;
1987 int i = 0;
1988 int ret = 0;
1989 dict_t *xdata = NULL;
1990 afr_fd_ctx_t *fd_ctx = NULL;
1991 unsigned char *wind_subvols = NULL;
1992
1993 priv = this->private;
1994 local = frame->local;
1995 wind_subvols = alloca0(priv->child_count);
1996
1997 afr_local_replies_wipe(local, priv);
1998
1999 if (local->fd) {
2000 fd_ctx = afr_fd_ctx_get(local->fd, this);
2001 if (!fd_ctx) {
2002 afr_inode_refresh_done(frame, this, EINVAL);
2003 return 0;
2004 }
2005 }
2006
2007 xdata = dict_new();
2008 if (!xdata) {
2009 afr_inode_refresh_done(frame, this, ENOMEM);
2010 return 0;
2011 }
2012
2013 ret = afr_xattr_req_prepare(this, xdata);
2014 if (ret != 0) {
2015 dict_unref(xdata);
2016 afr_inode_refresh_done(frame, this, -ret);
2017 return 0;
2018 }
2019
2020 ret = dict_set_sizen_str_sizen(xdata, "link-count", GF_XATTROP_INDEX_COUNT);
2021 if (ret) {
2022 gf_msg_debug(this->name, -ret, "Unable to set link-count in dict ");
2023 }
2024
2025 ret = dict_set_str_sizen(xdata, GLUSTERFS_INODELK_DOM_COUNT, this->name);
2026 if (ret) {
2027 gf_msg_debug(this->name, -ret,
2028 "Unable to set inodelk-dom-count in dict ");
2029 }
2030
2031 if (local->fd) {
2032 for (i = 0; i < priv->child_count; i++) {
2033 if (local->child_up[i] && fd_ctx->opened_on[i] == AFR_FD_OPENED)
2034 wind_subvols[i] = 1;
2035 }
2036 } else {
2037 memcpy(wind_subvols, local->child_up,
2038 sizeof(*local->child_up) * priv->child_count);
2039 }
2040
2041 local->call_count = AFR_COUNT(wind_subvols, priv->child_count);
2042
2043 call_count = local->call_count;
2044 if (!call_count) {
2045 dict_unref(xdata);
2046 if (local->fd && AFR_COUNT(local->child_up, priv->child_count))
2047 afr_inode_refresh_done(frame, this, EBADFD);
2048 else
2049 afr_inode_refresh_done(frame, this, ENOTCONN);
2050 return 0;
2051 }
2052 for (i = 0; i < priv->child_count; i++) {
2053 if (!wind_subvols[i])
2054 continue;
2055
2056 if (local->fd)
2057 afr_inode_refresh_subvol_with_fstat(frame, this, i, xdata);
2058 else
2059 afr_inode_refresh_subvol_with_lookup(
2060 frame, this, i, local->refreshinode, local->refreshgfid, xdata);
2061
2062 if (!--call_count)
2063 break;
2064 }
2065
2066 dict_unref(xdata);
2067
2068 return 0;
2069 }
2070
2071 int
2072 afr_inode_refresh(call_frame_t *frame, xlator_t *this, inode_t *inode,
2073 uuid_t gfid, afr_inode_refresh_cbk_t refreshfn)
2074 {
2075 afr_local_t *local = NULL;
2076
2077 local = frame->local;
2078
2079 local->refreshfn = refreshfn;
2080
2081 if (local->refreshinode) {
2082 inode_unref(local->refreshinode);
2083 local->refreshinode = NULL;
2084 }
2085
2086 local->refreshinode = inode_ref(inode);
2087
2088 if (gfid)
2089 gf_uuid_copy(local->refreshgfid, gfid);
2090 else
2091 gf_uuid_clear(local->refreshgfid);
2092
2093 afr_inode_refresh_do(frame, this);
2094
2095 return 0;
2096 }
2097
2098 int
2099 afr_xattr_req_prepare(xlator_t *this, dict_t *xattr_req)
2100 {
2101 int i = 0;
2102 afr_private_t *priv = NULL;
2103 int ret = 0;
2104
2105 priv = this->private;
2106
2107 for (i = 0; i < priv->child_count; i++) {
2108 ret = dict_set_uint64(xattr_req, priv->pending_key[i],
2109 AFR_NUM_CHANGE_LOGS * sizeof(int));
2110 if (ret < 0)
2111 gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
2112 "Unable to set dict value for %s", priv->pending_key[i]);
2113 /* 3 = data+metadata+entry */
2114 }
2115 ret = dict_set_uint64(xattr_req, AFR_DIRTY,
2116 AFR_NUM_CHANGE_LOGS * sizeof(int));
2117 if (ret) {
2118 gf_msg_debug(this->name, -ret,
2119 "failed to set dirty "
2120 "query flag");
2121 }
2122
2123 ret = dict_set_int32_sizen(xattr_req, "list-xattr", 1);
2124 if (ret) {
2125 gf_msg_debug(this->name, -ret, "Unable to set list-xattr in dict ");
2126 }
2127
2128 return ret;
2129 }
2130
2131 static int
2132 afr_lookup_xattr_req_prepare(afr_local_t *local, xlator_t *this,
2133 dict_t *xattr_req, loc_t *loc)
2134 {
2135 int ret = -ENOMEM;
2136
2137 if (!local->xattr_req)
2138 local->xattr_req = dict_new();
2139
2140 if (!local->xattr_req)
2141 goto out;
2142
2143 if (xattr_req && (xattr_req != local->xattr_req))
2144 dict_copy(xattr_req, local->xattr_req);
2145
2146 ret = afr_xattr_req_prepare(this, local->xattr_req);
2147
2148 ret = dict_set_uint64(local->xattr_req, GLUSTERFS_INODELK_COUNT, 0);
2149 if (ret < 0) {
2150 gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
2151 "%s: Unable to set dict value for %s", loc->path,
2152 GLUSTERFS_INODELK_COUNT);
2153 }
2154 ret = dict_set_uint64(local->xattr_req, GLUSTERFS_ENTRYLK_COUNT, 0);
2155 if (ret < 0) {
2156 gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
2157 "%s: Unable to set dict value for %s", loc->path,
2158 GLUSTERFS_ENTRYLK_COUNT);
2159 }
2160
2161 ret = dict_set_uint32(local->xattr_req, GLUSTERFS_PARENT_ENTRYLK, 0);
2162 if (ret < 0) {
2163 gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
2164 "%s: Unable to set dict value for %s", loc->path,
2165 GLUSTERFS_PARENT_ENTRYLK);
2166 }
2167
2168 ret = dict_set_sizen_str_sizen(local->xattr_req, "link-count",
2169 GF_XATTROP_INDEX_COUNT);
2170 if (ret) {
2171 gf_msg_debug(this->name, -ret, "Unable to set link-count in dict ");
2172 }
2173
2174 ret = 0;
2175 out:
2176 return ret;
2177 }
2178
2179 static int
2180 afr_least_pending_reads_child(afr_private_t *priv, unsigned char *readable)
2181 {
2182 int i = 0;
2183 int child = -1;
2184 int64_t read_iter = -1;
2185 int64_t pending_read = -1;
2186
2187 for (i = 0; i < priv->child_count; i++) {
2188 if (AFR_IS_ARBITER_BRICK(priv, i) || !readable[i])
2189 continue;
2190 read_iter = GF_ATOMIC_GET(priv->pending_reads[i]);
2191 if (child == -1 || read_iter < pending_read) {
2192 pending_read = read_iter;
2193 child = i;
2194 }
2195 }
2196
2197 return child;
2198 }
2199
2200 static int32_t
2201 afr_least_latency_child(afr_private_t *priv, unsigned char *readable)
2202 {
2203 int32_t i = 0;
2204 int child = -1;
2205
2206 for (i = 0; i < priv->child_count; i++) {
2207 if (AFR_IS_ARBITER_BRICK(priv, i) || !readable[i] ||
2208 priv->child_latency[i] < 0)
2209 continue;
2210
2211 if (child == -1 ||
2212 priv->child_latency[i] < priv->child_latency[child]) {
2213 child = i;
2214 }
2215 }
2216 return child;
2217 }
2218
2219 static int32_t
2220 afr_least_latency_times_pending_reads_child(afr_private_t *priv,
2221 unsigned char *readable)
2222 {
2223 int32_t i = 0;
2224 int child = -1;
2225 int64_t pending_read = 0;
2226 int64_t latency = -1;
2227 int64_t least_latency = -1;
2228
2229 for (i = 0; i < priv->child_count; i++) {
2230 if (AFR_IS_ARBITER_BRICK(priv, i) || !readable[i] ||
2231 priv->child_latency[i] < 0)
2232 continue;
2233
2234 pending_read = GF_ATOMIC_GET(priv->pending_reads[i]);
2235 latency = (pending_read + 1) * priv->child_latency[i];
2236
2237 if (child == -1 || latency < least_latency) {
2238 least_latency = latency;
2239 child = i;
2240 }
2241 }
2242 return child;
2243 }
2244
2245 static int
2246 afr_hash_child(afr_read_subvol_args_t *args, afr_private_t *priv,
2247 unsigned char *readable)
2248 {
2249 uuid_t gfid_copy = {
2250 0,
2251 };
2252 pid_t pid;
2253 int child = -1;
2254
2255 switch (priv->hash_mode) {
2256 case AFR_READ_POLICY_FIRST_UP:
2257 break;
2258 case AFR_READ_POLICY_GFID_HASH:
2259 gf_uuid_copy(gfid_copy, args->gfid);
2260 child = SuperFastHash((char *)gfid_copy, sizeof(gfid_copy)) %
2261 priv->child_count;
2262 break;
2263 case AFR_READ_POLICY_GFID_PID_HASH:
2264 if (args->ia_type != IA_IFDIR) {
2265 /*
2266 * Why getpid? Because it's one of the cheapest calls
2267 * available - faster than gethostname etc. - and
2268 * returns a constant-length value that's sure to be
2269 * shorter than a UUID. It's still very unlikely to be
2270 * the same across clients, so it still provides good
2271 * mixing. We're not trying for perfection here. All we
2272 * need is a low probability that multiple clients
2273 * won't converge on the same subvolume.
2274 */
2275 gf_uuid_copy(gfid_copy, args->gfid);
2276 pid = getpid();
2277 *(pid_t *)gfid_copy ^= pid;
2278 }
2279 child = SuperFastHash((char *)gfid_copy, sizeof(gfid_copy)) %
2280 priv->child_count;
2281 break;
2282 case AFR_READ_POLICY_LESS_LOAD:
2283 child = afr_least_pending_reads_child(priv, readable);
2284 break;
2285 case AFR_READ_POLICY_LEAST_LATENCY:
2286 child = afr_least_latency_child(priv, readable);
2287 break;
2288 case AFR_READ_POLICY_LOAD_LATENCY_HYBRID:
2289 child = afr_least_latency_times_pending_reads_child(priv, readable);
2290 break;
2291 }
2292
2293 return child;
2294 }
2295
2296 int
2297 afr_read_subvol_select_by_policy(inode_t *inode, xlator_t *this,
2298 unsigned char *readable,
2299 afr_read_subvol_args_t *args)
2300 {
2301 int i = 0;
2302 int read_subvol = -1;
2303 afr_private_t *priv = NULL;
2304 afr_read_subvol_args_t local_args = {
2305 0,
2306 };
2307
2308 priv = this->private;
2309
2310 /* first preference - explicitly specified or local subvolume */
2311 if (priv->read_child >= 0 && readable[priv->read_child])
2312 return priv->read_child;
2313
2314 if (inode_is_linked(inode)) {
2315 gf_uuid_copy(local_args.gfid, inode->gfid);
2316 local_args.ia_type = inode->ia_type;
2317 } else if (args) {
2318 local_args = *args;
2319 }
2320
2321 /* second preference - use hashed mode */
2322 read_subvol = afr_hash_child(&local_args, priv, readable);
2323 if (read_subvol >= 0 && readable[read_subvol])
2324 return read_subvol;
2325
2326 for (i = 0; i < priv->child_count; i++) {
2327 if (readable[i])
2328 return i;
2329 }
2330
2331 /* no readable subvolumes, either split brain or all subvols down */
2332
2333 return -1;
2334 }
2335
2336 static int
2337 afr_inode_read_subvol_type_get(inode_t *inode, xlator_t *this,
2338 unsigned char *readable, int *event_p, int type)
2339 {
2340 if (type == AFR_METADATA_TRANSACTION)
2341 return afr_inode_read_subvol_get(inode, this, 0, readable, event_p);
2342 else
2343 return afr_inode_read_subvol_get(inode, this, readable, 0, event_p);
2344 }
2345
2346 static void
2347 afr_readables_intersect_get(inode_t *inode, xlator_t *this, int *event,
2348 unsigned char *intersection)
2349 {
2350 afr_private_t *priv = NULL;
2351 unsigned char *data_readable = NULL;
2352 unsigned char *metadata_readable = NULL;
2353 unsigned char *intersect = NULL;
2354
2355 priv = this->private;
2356 data_readable = alloca0(priv->child_count);
2357 metadata_readable = alloca0(priv->child_count);
2358 intersect = alloca0(priv->child_count);
2359
2360 afr_inode_read_subvol_get(inode, this, data_readable, metadata_readable,
2361 event);
2362
2363 AFR_INTERSECT(intersect, data_readable, metadata_readable,
2364 priv->child_count);
2365 if (intersection)
2366 memcpy(intersection, intersect,
2367 sizeof(*intersection) * priv->child_count);
2368 }
2369
2370 int
2371 afr_read_subvol_get(inode_t *inode, xlator_t *this, int *subvol_p,
2372 unsigned char *readables, int *event_p,
2373 afr_transaction_type type, afr_read_subvol_args_t *args)
2374 {
2375 afr_private_t *priv = NULL;
2376 unsigned char *readable = NULL;
2377 unsigned char *intersection = NULL;
2378 int subvol = -1;
2379 int event = 0;
2380
2381 priv = this->private;
2382
2383 readable = alloca0(priv->child_count);
2384 intersection = alloca0(priv->child_count);
2385
2386 afr_inode_read_subvol_type_get(inode, this, readable, &event, type);
2387
2388 afr_readables_intersect_get(inode, this, &event, intersection);
2389
2390 if (AFR_COUNT(intersection, priv->child_count) > 0)
2391 subvol = afr_read_subvol_select_by_policy(inode, this, intersection,
2392 args);
2393 else
2394 subvol = afr_read_subvol_select_by_policy(inode, this, readable, args);
2395 if (subvol_p)
2396 *subvol_p = subvol;
2397 if (event_p)
2398 *event_p = event;
2399 if (readables)
2400 memcpy(readables, readable, sizeof(*readables) * priv->child_count);
2401 return subvol;
2402 }
2403
2404 static void
2405 afr_local_transaction_cleanup(afr_local_t *local, unsigned int child_count)
2406 {
2407 int i = 0;
2408
2409 afr_matrix_cleanup(local->pending, child_count);
2410
2411 afr_lockees_cleanup(&local->internal_lock);
2412
2413 GF_FREE(local->transaction.pre_op);
2414
2415 GF_FREE(local->transaction.pre_op_sources);
2416 if (local->transaction.changelog_xdata) {
2417 for (i = 0; i < child_count; i++) {
2418 if (!local->transaction.changelog_xdata[i])
2419 continue;
2420 dict_unref(local->transaction.changelog_xdata[i]);
2421 }
2422 GF_FREE(local->transaction.changelog_xdata);
2423 }
2424
2425 GF_FREE(local->transaction.failed_subvols);
2426
2427 GF_FREE(local->transaction.basename);
2428 GF_FREE(local->transaction.new_basename);
2429
2430 loc_wipe(&local->transaction.parent_loc);
2431 loc_wipe(&local->transaction.new_parent_loc);
2432 }
2433
2434 void
2435 afr_reply_wipe(struct afr_reply *reply)
2436 {
2437 if (reply->xdata) {
2438 dict_unref(reply->xdata);
2439 reply->xdata = NULL;
2440 }
2441
2442 if (reply->xattr) {
2443 dict_unref(reply->xattr);
2444 reply->xattr = NULL;
2445 }
2446 }
2447
2448 void
2449 afr_replies_wipe(struct afr_reply *replies, int count)
2450 {
2451 int i = 0;
2452
2453 for (i = 0; i < count; i++) {
2454 afr_reply_wipe(&replies[i]);
2455 }
2456 }
2457
2458 void
2459 afr_local_replies_wipe(afr_local_t *local, afr_private_t *priv)
2460 {
2461 if (!local->replies)
2462 return;
2463
2464 afr_replies_wipe(local->replies, priv->child_count);
2465
2466 memset(local->replies, 0, sizeof(*local->replies) * priv->child_count);
2467 }
2468
2469 static gf_boolean_t
2470 afr_fop_lock_is_unlock(call_frame_t *frame)
2471 {
2472 afr_local_t *local = frame->local;
2473 switch (local->op) {
2474 case GF_FOP_INODELK:
2475 case GF_FOP_FINODELK:
2476 if ((F_UNLCK == local->cont.inodelk.in_flock.l_type) &&
2477 (local->cont.inodelk.in_cmd == F_SETLKW ||
2478 local->cont.inodelk.in_cmd == F_SETLK))
2479 return _gf_true;
2480 break;
2481 case GF_FOP_ENTRYLK:
2482 case GF_FOP_FENTRYLK:
2483 if (ENTRYLK_UNLOCK == local->cont.entrylk.in_cmd)
2484 return _gf_true;
2485 break;
2486 default:
2487 return _gf_false;
2488 }
2489 return _gf_false;
2490 }
2491
2492 static gf_boolean_t
2493 afr_lk_is_unlock(int32_t cmd, struct gf_flock *flock)
2494 {
2495 switch (cmd) {
2496 case F_RESLK_UNLCK:
2497 return _gf_true;
2498 break;
2499
2500 #if F_SETLKW != F_SETLKW64
2501 case F_SETLKW64:
2502 #endif
2503 case F_SETLKW:
2504
2505 #if F_SETLK != F_SETLK64
2506 case F_SETLK64:
2507 #endif
2508 case F_SETLK:
2509 if (F_UNLCK == flock->l_type)
2510 return _gf_true;
2511 break;
2512 default:
2513 return _gf_false;
2514 }
2515 return _gf_false;
2516 }
2517
2518 void
2519 afr_handle_inconsistent_fop(call_frame_t *frame, int32_t *op_ret,
2520 int32_t *op_errno)
2521 {
2522 afr_private_t *priv = NULL;
2523 afr_local_t *local = NULL;
2524
2525 if (!frame || !frame->this || !frame->local || !frame->this->private)
2526 return;
2527
2528 if (*op_ret < 0)
2529 return;
2530
2531 /* Failing inodelk/entrylk/lk here is not a good idea because we
2532 * need to cleanup the locks on the other bricks if we choose to fail
2533 * the fop here. The brick may go down just after unwind happens as well
2534 * so anyways the fop will fail when the next fop is sent so leaving
2535 * it like this for now.*/
2536 local = frame->local;
2537 switch (local->op) {
2538 case GF_FOP_LOOKUP:
2539 case GF_FOP_INODELK:
2540 case GF_FOP_FINODELK:
2541 case GF_FOP_ENTRYLK:
2542 case GF_FOP_FENTRYLK:
2543 case GF_FOP_LK:
2544 return;
2545 default:
2546 break;
2547 }
2548
2549 priv = frame->this->private;
2550 if (!priv->consistent_io)
2551 return;
2552
2553 if (local->event_generation &&
2554 (local->event_generation != priv->event_generation))
2555 goto inconsistent;
2556
2557 return;
2558 inconsistent:
2559 *op_ret = -1;
2560 *op_errno = ENOTCONN;
2561 }
2562
2563 void
2564 afr_local_cleanup(afr_local_t *local, xlator_t *this)
2565 {
2566 afr_private_t *priv = NULL;
2567
2568 if (!local)
2569 return;
2570 priv = this->private;
2571
2572 syncbarrier_destroy(&local->barrier);
2573
2574 afr_local_transaction_cleanup(local, priv->child_count);
2575
2576 loc_wipe(&local->loc);
2577 loc_wipe(&local->newloc);
2578
2579 if (local->fd)
2580 fd_unref(local->fd);
2581
2582 if (local->xattr_req)
2583 dict_unref(local->xattr_req);
2584
2585 if (local->xattr_rsp)
2586 dict_unref(local->xattr_rsp);
2587
2588 if (local->dict)
2589 dict_unref(local->dict);
2590
2591 afr_local_replies_wipe(local, priv);
2592 GF_FREE(local->replies);
2593
2594 GF_FREE(local->child_up);
2595
2596 GF_FREE(local->read_attempted);
2597
2598 GF_FREE(local->readable);
2599 GF_FREE(local->readable2);
2600
2601 if (local->inode)
2602 inode_unref(local->inode);
2603
2604 if (local->parent)
2605 inode_unref(local->parent);
2606
2607 if (local->parent2)
2608 inode_unref(local->parent2);
2609
2610 if (local->refreshinode)
2611 inode_unref(local->refreshinode);
2612
2613 { /* getxattr */
2614 GF_FREE(local->cont.getxattr.name);
2615 }
2616
2617 { /* lk */
2618 GF_FREE(local->cont.lk.locked_nodes);
2619 GF_FREE(local->cont.lk.dom_locked_nodes);
2620 GF_FREE(local->cont.lk.dom_lock_op_ret);
2621 GF_FREE(local->cont.lk.dom_lock_op_errno);
2622 }
2623
2624 { /* create */
2625 if (local->cont.create.fd)
2626 fd_unref(local->cont.create.fd);
2627 if (local->cont.create.params)
2628 dict_unref(local->cont.create.params);
2629 }
2630
2631 { /* mknod */
2632 if (local->cont.mknod.params)
2633 dict_unref(local->cont.mknod.params);
2634 }
2635
2636 { /* mkdir */
2637 if (local->cont.mkdir.params)
2638 dict_unref(local->cont.mkdir.params);
2639 }
2640
2641 { /* symlink */
2642 if (local->cont.symlink.params)
2643 dict_unref(local->cont.symlink.params);
2644 }
2645
2646 { /* writev */
2647 GF_FREE(local->cont.writev.vector);
2648 if (local->cont.writev.iobref)
2649 iobref_unref(local->cont.writev.iobref);
2650 }
2651
2652 { /* setxattr */
2653 if (local->cont.setxattr.dict)
2654 dict_unref(local->cont.setxattr.dict);
2655 }
2656
2657 { /* fsetxattr */
2658 if (local->cont.fsetxattr.dict)
2659 dict_unref(local->cont.fsetxattr.dict);
2660 }
2661
2662 { /* removexattr */
2663 GF_FREE(local->cont.removexattr.name);
2664 }
2665 { /* xattrop */
2666 if (local->cont.xattrop.xattr)
2667 dict_unref(local->cont.xattrop.xattr);
2668 }
2669 { /* symlink */
2670 GF_FREE(local->cont.symlink.linkpath);
2671 }
2672
2673 { /* opendir */
2674 GF_FREE(local->cont.opendir.checksum);
2675 }
2676
2677 { /* open */
2678 if (local->cont.open.fd)
2679 fd_unref(local->cont.open.fd);
2680 }
2681
2682 { /* readdirp */
2683 if (local->cont.readdir.dict)
2684 dict_unref(local->cont.readdir.dict);
2685 }
2686
2687 { /* inodelk */
2688 GF_FREE(local->cont.inodelk.volume);
2689 if (local->cont.inodelk.xdata)
2690 dict_unref(local->cont.inodelk.xdata);
2691 }
2692
2693 { /* entrylk */
2694 GF_FREE(local->cont.entrylk.volume);
2695 GF_FREE(local->cont.entrylk.basename);
2696 if (local->cont.entrylk.xdata)
2697 dict_unref(local->cont.entrylk.xdata);
2698 }
2699
2700 GF_FREE(local->need_open);
2701
2702 if (local->xdata_req)
2703 dict_unref(local->xdata_req);
2704
2705 if (local->xdata_rsp)
2706 dict_unref(local->xdata_rsp);
2707 }
2708
2709 int
2710 afr_frame_return(call_frame_t *frame)
2711 {
2712 afr_local_t *local = NULL;
2713 int call_count = 0;
2714
2715 local = frame->local;
2716
2717 LOCK(&frame->lock);
2718 {
2719 call_count = --local->call_count;
2720 }
2721 UNLOCK(&frame->lock);
2722
2723 return call_count;
2724 }
2725
2726 static char *afr_ignore_xattrs[] = {GF_SELINUX_XATTR_KEY, QUOTA_SIZE_KEY,
2727 SQUOTA_SIZE_KEY, SQUOTA_LIMIT_KEY, NULL};
2728
2729 gf_boolean_t
2730 afr_is_xattr_ignorable(char *key)
2731 {
2732 int i = 0;
2733
2734 if (!strncmp(key, AFR_XATTR_PREFIX, SLEN(AFR_XATTR_PREFIX)))
2735 return _gf_true;
2736 for (i = 0; afr_ignore_xattrs[i]; i++) {
2737 if (!strcmp(key, afr_ignore_xattrs[i]))
2738 return _gf_true;
2739 }
2740 return _gf_false;
2741 }
2742
2743 static gf_boolean_t
2744 afr_xattr_match_needed(dict_t *this, char *key1, data_t *value1, void *data)
2745 {
2746 gf_boolean_t *is_arbiter = NULL;
2747
2748 is_arbiter = data;
2749 /* Ignore all non-disk (i.e. virtual) xattrs right away. */
2750 if (!gf_is_valid_xattr_namespace(key1))
2751 return _gf_false;
2752
2753 if (is_arbiter && *is_arbiter && !strcmp(key1, GF_XATTR_MDATA_KEY)) {
2754 return _gf_false;
2755 }
2756 /* Ignore on-disk xattrs that AFR doesn't need to heal. */
2757 if (!afr_is_xattr_ignorable(key1))
2758 return _gf_true;
2759
2760 return _gf_false;
2761 }
2762
2763 gf_boolean_t
2764 afr_xattrs_are_equal(dict_t *dict1, dict_t *dict2, gf_boolean_t is_arbiter)
2765 {
2766 return are_dicts_equal(dict1, dict2, afr_xattr_match_needed, &is_arbiter,
2767 NULL);
2768 }
2769
2770 static int
2771 afr_get_parent_read_subvol(xlator_t *this, inode_t *parent,
2772 struct afr_reply *replies, unsigned char *readable)
2773 {
2774 int i = 0;
2775 int par_read_subvol = -1;
2776 int par_read_subvol_iter = -1;
2777 afr_private_t *priv = NULL;
2778
2779 priv = this->private;
2780
2781 if (parent)
2782 par_read_subvol = afr_data_subvol_get(parent, this, NULL, NULL, NULL,
2783 NULL);
2784
2785 for (i = 0; i < priv->child_count; i++) {
2786 if (!replies[i].valid)
2787 continue;
2788
2789 if (replies[i].op_ret < 0)
2790 continue;
2791
2792 if (par_read_subvol_iter == -1) {
2793 par_read_subvol_iter = i;
2794 continue;
2795 }
2796
2797 if ((par_read_subvol_iter != par_read_subvol) && readable[i])
2798 par_read_subvol_iter = i;
2799
2800 if (i == par_read_subvol)
2801 par_read_subvol_iter = i;
2802 }
2803 /* At the end of the for-loop, the only reason why @par_read_subvol_iter
2804 * could be -1 is when this LOOKUP has failed on all sub-volumes.
2805 * So it is okay to send an arbitrary subvolume (0 in this case)
2806 * as parent read subvol.
2807 */
2808 if (par_read_subvol_iter == -1)
2809 par_read_subvol_iter = 0;
2810
2811 return par_read_subvol_iter;
2812 }
2813
2814 static int
2815 afr_read_subvol_decide(inode_t *inode, xlator_t *this,
2816 afr_read_subvol_args_t *args, unsigned char *readable)
2817 {
2818 int event = 0;
2819 afr_private_t *priv = NULL;
2820 unsigned char *intersection = NULL;
2821
2822 priv = this->private;
2823 intersection = alloca0(priv->child_count);
2824
2825 afr_readables_intersect_get(inode, this, &event, intersection);
2826
2827 if (AFR_COUNT(intersection, priv->child_count) <= 0) {
2828 /* TODO: If we have one brick with valid data_readable and
2829 * another with metadata_readable, try to send an iatt with
2830 * valid bits from both.*/
2831 return -1;
2832 }
2833
2834 memcpy(readable, intersection, sizeof(*readable) * priv->child_count);
2835
2836 return afr_read_subvol_select_by_policy(inode, this, intersection, args);
2837 }
2838
2839 static inline int
2840 afr_first_up_child(call_frame_t *frame, xlator_t *this)
2841 {
2842 afr_private_t *priv = NULL;
2843 afr_local_t *local = NULL;
2844 int i = 0;
2845
2846 local = frame->local;
2847 priv = this->private;
2848
2849 for (i = 0; i < priv->child_count; i++)
2850 if (local->replies[i].valid && local->replies[i].op_ret == 0)
2851 return i;
2852 return -1;
2853 }
2854
2855 static void
2856 afr_attempt_readsubvol_set(call_frame_t *frame, xlator_t *this,
2857 unsigned char *success_replies,
2858 unsigned char *data_readable, int *read_subvol)
2859 {
2860 afr_private_t *priv = NULL;
2861 afr_local_t *local = NULL;
2862 int spb_subvol = -1;
2863 int child_count = -1;
2864
2865 if (*read_subvol != -1)
2866 return;
2867
2868 priv = this->private;
2869 local = frame->local;
2870 child_count = priv->child_count;
2871
2872 afr_split_brain_read_subvol_get(local->inode, this, frame, &spb_subvol);
2873 if ((spb_subvol >= 0) &&
2874 (AFR_COUNT(success_replies, child_count) == child_count)) {
2875 *read_subvol = spb_subvol;
2876 } else if (!priv->quorum_count ||
2877 frame->root->pid == GF_CLIENT_PID_GLFS_HEAL) {
2878 *read_subvol = afr_first_up_child(frame, this);
2879 } else if (priv->quorum_count &&
2880 afr_has_quorum(data_readable, priv, NULL)) {
2881 /* read_subvol is guaranteed to be valid if we hit this path. */
2882 *read_subvol = afr_first_up_child(frame, this);
2883 } else {
2884 /* If quorum is enabled and we do not have a
2885 readable yet, it means all good copies are down.
2886 */
2887 local->op_ret = -1;
2888 local->op_errno = ENOTCONN;
2889 gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_READ_SUBVOL_ERROR,
2890 "no read "
2891 "subvols for %s",
2892 local->loc.path);
2893 }
2894 if (*read_subvol >= 0)
2895 dict_del_sizen(local->replies[*read_subvol].xdata, GF_CONTENT_KEY);
2896 }
2897
2898 static void
2899 afr_lookup_done(call_frame_t *frame, xlator_t *this)
2900 {
2901 afr_private_t *priv = NULL;
2902 afr_local_t *local = NULL;
2903 int i = -1;
2904 int op_errno = 0;
2905 int read_subvol = 0;
2906 int par_read_subvol = 0;
2907 int ret = -1;
2908 unsigned char *readable = NULL;
2909 unsigned char *success_replies = NULL;
2910 int event = 0;
2911 struct afr_reply *replies = NULL;
2912 uuid_t read_gfid = {
2913 0,
2914 };
2915 gf_boolean_t locked_entry = _gf_false;
2916 gf_boolean_t in_flight_create = _gf_false;
2917 gf_boolean_t can_interpret = _gf_true;
2918 inode_t *parent = NULL;
2919 ia_type_t ia_type = IA_INVAL;
2920 afr_read_subvol_args_t args = {
2921 0,
2922 };
2923 char *gfid_heal_msg = NULL;
2924
2925 priv = this->private;
2926 local = frame->local;
2927 replies = local->replies;
2928 parent = local->loc.parent;
2929
2930 locked_entry = afr_is_possibly_under_txn(AFR_ENTRY_TRANSACTION, local,
2931 priv->child_count);
2932
2933 readable = alloca0(priv->child_count);
2934 success_replies = alloca0(priv->child_count);
2935
2936 afr_inode_read_subvol_get(parent, this, readable, NULL, &event);
2937 par_read_subvol = afr_get_parent_read_subvol(this, parent, replies,
2938 readable);
2939
2940 /* First, check if we have a gfid-change from somewhere,
2941 If so, propagate that so that a fresh lookup can be
2942 issued
2943 */
2944 if (local->cont.lookup.needs_fresh_lookup) {
2945 local->op_ret = -1;
2946 local->op_errno = ESTALE;
2947 goto error;
2948 }
2949
2950 op_errno = afr_final_errno(frame->local, this->private);
2951 local->op_errno = op_errno;
2952
2953 read_subvol = -1;
2954 afr_fill_success_replies(local, priv, success_replies);
2955
2956 for (i = 0; i < priv->child_count; i++) {
2957 if (!replies[i].valid)
2958 continue;
2959
2960 if (replies[i].op_ret == -1) {
2961 if (locked_entry && replies[i].op_errno == ENOENT) {
2962 in_flight_create = _gf_true;
2963 }
2964 continue;
2965 }
2966
2967 if (read_subvol == -1 || !readable[read_subvol]) {
2968 read_subvol = i;
2969 gf_uuid_copy(read_gfid, replies[i].poststat.ia_gfid);
2970 ia_type = replies[i].poststat.ia_type;
2971 local->op_ret = 0;
2972 }
2973 }
2974
2975 if (in_flight_create && !afr_has_quorum(success_replies, priv, NULL)) {
2976 local->op_ret = -1;
2977 local->op_errno = ENOENT;
2978 goto error;
2979 }
2980
2981 if (read_subvol == -1)
2982 goto error;
2983 /* We now have a read_subvol, which is readable[] (if there
2984 were any). Next we look for GFID mismatches. We don't
2985 consider a GFID mismatch as an error if read_subvol is
2986 readable[] but the mismatching GFID subvol is not.
2987 */
2988 for (i = 0; i < priv->child_count; i++) {
2989 if (!replies[i].valid || replies[i].op_ret == -1) {
2990 continue;
2991 }
2992
2993 if (!gf_uuid_compare(replies[i].poststat.ia_gfid, read_gfid))
2994 continue;
2995
2996 can_interpret = _gf_false;
2997
2998 if (locked_entry)
2999 continue;
3000
3001 /* Now GFIDs mismatch. It's OK as long as this subvol
3002 is not readable[] but read_subvol is */
3003 if (readable[read_subvol] && !readable[i])
3004 continue;
3005
3006 /* If we were called from glfsheal and there is still a gfid
3007 * mismatch, succeed the lookup and let glfsheal print the
3008 * response via gfid-heal-msg.*/
3009 if (!dict_get_str_sizen(local->xattr_rsp, "gfid-heal-msg",
3010 &gfid_heal_msg))
3011 goto cant_interpret;
3012
3013 /* LOG ERROR */
3014 local->op_ret = -1;
3015 local->op_errno = EIO;
3016 goto error;
3017 }
3018
3019 /* Forth, for the finalized GFID, pick the best subvolume
3020 to return stats from.
3021 */
3022 read_subvol = -1;
3023 memset(readable, 0, sizeof(*readable) * priv->child_count);
3024 if (can_interpret) {
3025 if (!afr_has_quorum(success_replies, priv, NULL))
3026 goto cant_interpret;
3027 /* It is safe to call afr_replies_interpret() because we have
3028 a response from all the UP subvolumes and all of them resolved
3029 to the same GFID
3030 */
3031 gf_uuid_copy(args.gfid, read_gfid);
3032 args.ia_type = ia_type;
3033 ret = afr_replies_interpret(local, this, local->inode, NULL);
3034 read_subvol = afr_read_subvol_decide(local->inode, this, &args,
3035 readable);
3036 if (read_subvol == -1)
3037 goto cant_interpret;
3038 if (ret) {
3039 afr_inode_need_refresh_set(local->inode, this);
3040 dict_del_sizen(local->replies[read_subvol].xdata, GF_CONTENT_KEY);
3041 }
3042 } else {
3043 cant_interpret:
3044 afr_attempt_readsubvol_set(frame, this, success_replies, readable,
3045 &read_subvol);
3046 if (read_subvol == -1) {
3047 goto error;
3048 }
3049 }
3050
3051 afr_handle_quota_size(local, this);
3052
3053 afr_set_need_heal(this, local);
3054 if (AFR_IS_ARBITER_BRICK(priv, read_subvol) && local->op_ret == 0) {
3055 local->op_ret = -1;
3056 local->op_errno = ENOTCONN;
3057 gf_msg_debug(this->name, 0,
3058 "Arbiter cannot be a read subvol "
3059 "for %s",
3060 local->loc.path);
3061 goto error;
3062 }
3063
3064 ret = dict_get_str_sizen(local->xattr_rsp, "gfid-heal-msg", &gfid_heal_msg);
3065 if (!ret) {
3066 ret = dict_set_str_sizen(local->replies[read_subvol].xdata,
3067 "gfid-heal-msg", gfid_heal_msg);
3068 if (ret) {
3069 gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_DICT_SET_FAILED,
3070 "Error setting gfid-heal-msg dict");
3071 local->op_ret = -1;
3072 local->op_errno = ENOMEM;
3073 }
3074 }
3075
3076 AFR_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
3077 local->inode, &local->replies[read_subvol].poststat,
3078 local->replies[read_subvol].xdata,
3079 &local->replies[par_read_subvol].postparent);
3080 return;
3081
3082 error:
3083 AFR_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno, NULL, NULL,
3084 NULL, NULL);
3085 }
3086
3087 /*
3088 * During a lookup, some errors are more "important" than
3089 * others in that they must be given higher priority while
3090 * returning to the user.
3091 *
3092 * The hierarchy is ENODATA > ENOENT > ESTALE > ENOSPC others
3093 */
3094
3095 int
3096 afr_higher_errno(int32_t old_errno, int32_t new_errno)
3097 {
3098 if (old_errno == ENODATA || new_errno == ENODATA)
3099 return ENODATA;
3100 if (old_errno == ENOENT || new_errno == ENOENT)
3101 return ENOENT;
3102 if (old_errno == ESTALE || new_errno == ESTALE)
3103 return ESTALE;
3104 if (old_errno == ENOSPC || new_errno == ENOSPC)
3105 return ENOSPC;
3106
3107 return new_errno;
3108 }
3109
3110 int
3111 afr_final_errno(afr_local_t *local, afr_private_t *priv)
3112 {
3113 int i = 0;
3114 int op_errno = 0;
3115 int tmp_errno = 0;
3116
3117 for (i = 0; i < priv->child_count; i++) {
3118 if (!local->replies[i].valid)
3119 continue;
3120 if (local->replies[i].op_ret >= 0)
3121 continue;
3122 tmp_errno = local->replies[i].op_errno;
3123 op_errno = afr_higher_errno(op_errno, tmp_errno);
3124 }
3125
3126 return op_errno;
3127 }
3128
3129 static int32_t
3130 afr_local_discovery_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
3131 int32_t op_ret, int32_t op_errno, dict_t *dict,
3132 dict_t *xdata)
3133 {
3134 int ret = 0;
3135 char *pathinfo = NULL;
3136 gf_boolean_t is_local = _gf_false;
3137 afr_private_t *priv = NULL;
3138 int32_t child_index = -1;
3139
3140 if (op_ret != 0) {
3141 goto out;
3142 }
3143
3144 priv = this->private;
3145 child_index = (int32_t)(long)cookie;
3146
3147 ret = dict_get_str_sizen(dict, GF_XATTR_PATHINFO_KEY, &pathinfo);
3148 if (ret != 0) {
3149 goto out;
3150 }
3151
3152 ret = glusterfs_is_local_pathinfo(pathinfo, &is_local);
3153 if (ret) {
3154 goto out;
3155 }
3156
3157 /*
3158 * Note that one local subvolume will override another here. The only
3159 * way to avoid that would be to retain extra information about whether
3160 * the previous read_child is local, and it's just not worth it. Even
3161 * the slowest local subvolume is far preferable to a remote one.
3162 */
3163 if (is_local) {
3164 priv->local[child_index] = 1;
3165 /* Don't set arbiter as read child. */
3166 if (AFR_IS_ARBITER_BRICK(priv, child_index))
3167 goto out;
3168 gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_LOCAL_CHILD,
3169 "selecting local read_child %s",
3170 priv->children[child_index]->name);
3171
3172 priv->read_child = child_index;
3173 }
3174 out:
3175 STACK_DESTROY(frame->root);
3176 return 0;
3177 }
3178
3179 static void
3180 afr_attempt_local_discovery(xlator_t *this, int32_t child_index)
3181 {
3182 call_frame_t *newframe = NULL;
3183 loc_t tmploc = {
3184 0,
3185 };
3186 afr_private_t *priv = this->private;
3187
3188 newframe = create_frame(this, this->ctx->pool);
3189 if (!newframe) {
3190 return;
3191 }
3192
3193 tmploc.gfid[sizeof(tmploc.gfid) - 1] = 1;
3194 STACK_WIND_COOKIE(newframe, afr_local_discovery_cbk,
3195 (void *)(long)child_index, priv->children[child_index],
3196 priv->children[child_index]->fops->getxattr, &tmploc,
3197 GF_XATTR_PATHINFO_KEY, NULL);
3198 }
3199
3200 static int
3201 afr_lookup_sh_metadata_wrap(void *opaque)
3202 {
3203 call_frame_t *frame = opaque;
3204 afr_local_t *local = NULL;
3205 xlator_t *this = NULL;
3206 inode_t *inode = NULL;
3207 afr_private_t *priv = NULL;
3208 struct afr_reply *replies = NULL;
3209 int i = 0, first = -1;
3210 int ret = -1;
3211 dict_t *dict = NULL;
3212
3213 local = frame->local;
3214 this = frame->this;
3215 priv = this->private;
3216 replies = local->replies;
3217
3218 for (i = 0; i < priv->child_count; i++) {
3219 if (!replies[i].valid || replies[i].op_ret == -1)
3220 continue;
3221 first = i;
3222 break;
3223 }
3224 if (first == -1)
3225 goto out;
3226
3227 if (afr_selfheal_metadata_by_stbuf(this, &replies[first].poststat))
3228 goto out;
3229
3230 afr_local_replies_wipe(local, this->private);
3231
3232 dict = dict_new();
3233 if (!dict)
3234 goto out;
3235 if (local->xattr_req) {
3236 dict_copy(local->xattr_req, dict);
3237 }
3238
3239 ret = dict_set_sizen_str_sizen(dict, "link-count", GF_XATTROP_INDEX_COUNT);
3240 if (ret) {
3241 gf_msg_debug(this->name, -ret, "Unable to set link-count in dict ");
3242 }
3243
3244 if (loc_is_nameless(&local->loc)) {
3245 ret = afr_selfheal_unlocked_discover_on(frame, local->inode,
3246 local->loc.gfid, local->replies,
3247 local->child_up, dict);
3248 } else {
3249 inode = afr_selfheal_unlocked_lookup_on(frame, local->loc.parent,
3250 local->loc.name, local->replies,
3251 local->child_up, dict);
3252 }
3253 if (inode)
3254 inode_unref(inode);
3255 out:
3256 if (loc_is_nameless(&local->loc))
3257 afr_discover_done(frame, this);
3258 else
3259 afr_lookup_done(frame, this);
3260
3261 if (dict)
3262 dict_unref(dict);
3263
3264 return 0;
3265 }
3266
3267 gf_boolean_t
3268 afr_is_pending_set(xlator_t *this, dict_t *xdata, int type)
3269 {
3270 int idx = -1;
3271 afr_private_t *priv = NULL;
3272 void *pending_raw = NULL;
3273 int i = 0;
3274
3275 priv = this->private;
3276 idx = afr_index_for_transaction_type(type);
3277
3278 if (dict_get_ptr(xdata, AFR_DIRTY, &pending_raw) == 0) {
3279 if (pending_raw) {
3280 if (*((int *)pending_raw + idx))
3281 return _gf_true;
3282 }
3283 }
3284
3285 for (i = 0; i < priv->child_count; i++) {
3286 if (dict_get_ptr(xdata, priv->pending_key[i], &pending_raw))
3287 continue;
3288 if (!pending_raw)
3289 continue;
3290 if (*((int *)pending_raw + idx))
3291 return _gf_true;
3292 }
3293
3294 return _gf_false;
3295 }
3296
3297 static gf_boolean_t
3298 afr_can_start_metadata_self_heal(call_frame_t *frame, xlator_t *this)
3299 {
3300 afr_local_t *local = NULL;
3301 afr_private_t *priv = NULL;
3302 struct afr_reply *replies = NULL;
3303 int i = 0, first = -1;
3304 gf_boolean_t start = _gf_false;
3305 struct iatt stbuf = {
3306 0,
3307 };
3308
3309 local = frame->local;
3310 replies = local->replies;
3311 priv = this->private;
3312
3313 if (!priv->metadata_self_heal)
3314 return _gf_false;
3315
3316 for (i = 0; i < priv->child_count; i++) {
3317 if (!replies[i].valid || replies[i].op_ret == -1)
3318 continue;
3319 if (first == -1) {
3320 first = i;
3321 stbuf = replies[i].poststat;
3322 continue;
3323 }
3324
3325 if (afr_is_pending_set(this, replies[i].xdata,
3326 AFR_METADATA_TRANSACTION)) {
3327 /* Let shd do the heal so that lookup is not blocked
3328 * on getting metadata lock/doing the heal */
3329 start = _gf_false;
3330 break;
3331 }
3332
3333 if (gf_uuid_compare(stbuf.ia_gfid, replies[i].poststat.ia_gfid)) {
3334 start = _gf_false;
3335 break;
3336 }
3337 if (!IA_EQUAL(stbuf, replies[i].poststat, type)) {
3338 start = _gf_false;
3339 break;
3340 }
3341
3342 /*Check if iattrs need heal*/
3343 if ((!IA_EQUAL(stbuf, replies[i].poststat, uid)) ||
3344 (!IA_EQUAL(stbuf, replies[i].poststat, gid)) ||
3345 (!IA_EQUAL(stbuf, replies[i].poststat, prot))) {
3346 start = _gf_true;
3347 continue;
3348 }
3349
3350 /*Check if xattrs need heal*/
3351 if (!afr_xattrs_are_equal(
3352 replies[first].xdata, replies[i].xdata,
3353 AFR_IS_ARBITER_BRICK(priv, i) ? _gf_true : _gf_false))
3354 start = _gf_true;
3355 }
3356
3357 return start;
3358 }
3359
3360 int
3361 afr_lookup_metadata_heal_check(call_frame_t *frame, xlator_t *this)
3362
3363 {
3364 call_frame_t *heal = NULL;
3365 afr_local_t *local = NULL;
3366 int ret = 0;
3367
3368 local = frame->local;
3369 if (!afr_can_start_metadata_self_heal(frame, this))
3370 goto out;
3371
3372 heal = afr_frame_create(this, &ret);
3373 if (!heal) {
3374 ret = -ret;
3375 goto out;
3376 }
3377
3378 ret = synctask_new(this->ctx->env, afr_lookup_sh_metadata_wrap,
3379 afr_refresh_selfheal_done, heal, frame);
3380 if (ret)
3381 goto out;
3382 return ret;
3383 out:
3384 if (loc_is_nameless(&local->loc))
3385 afr_discover_done(frame, this);
3386 else
3387 afr_lookup_done(frame, this);
3388 if (heal)
3389 AFR_STACK_DESTROY(heal);
3390 return ret;
3391 }
3392
3393 int
3394 afr_lookup_selfheal_wrap(void *opaque)
3395 {
3396 int ret = 0;
3397 call_frame_t *frame = opaque;
3398 afr_local_t *local = NULL;
3399 xlator_t *this = NULL;
3400 inode_t *inode = NULL;
3401 uuid_t pargfid = {
3402 0,
3403 };
3404
3405 local = frame->local;
3406 this = frame->this;
3407 loc_pargfid(&local->loc, pargfid);
3408 if (!local->xattr_rsp)
3409 local->xattr_rsp = dict_new();
3410
3411 ret = afr_selfheal_name(frame->this, pargfid, local->loc.name,
3412 &local->cont.lookup.gfid_req, local->xattr_req,
3413 local->xattr_rsp);
3414 if (ret == -EIO)
3415 goto unwind;
3416
3417 afr_local_replies_wipe(local, this->private);
3418
3419 inode = afr_selfheal_unlocked_lookup_on(frame, local->loc.parent,
3420 local->loc.name, local->replies,
3421 local->child_up, local->xattr_req);
3422 if (inode)
3423 inode_unref(inode);
3424
3425 afr_lookup_metadata_heal_check(frame, this);
3426 return 0;
3427
3428 unwind:
3429 AFR_STACK_UNWIND(lookup, frame, -1, EIO, NULL, NULL, local->xattr_rsp,
3430 NULL);
3431 return 0;
3432 }
3433
3434 static int
3435 afr_lookup_entry_heal(call_frame_t *frame, xlator_t *this)
3436 {
3437 afr_local_t *local = NULL;
3438 afr_private_t *priv = NULL;
3439 call_frame_t *heal = NULL;
3440 int i = 0, first = -1;
3441 gf_boolean_t name_state_mismatch = _gf_false;
3442 struct afr_reply *replies = NULL;
3443 int ret = 0;
3444 unsigned char *par_readables = NULL;
3445 unsigned char *success = NULL;
3446 int32_t op_errno = 0;
3447 uuid_t gfid = {0};
3448
3449 local = frame->local;
3450 replies = local->replies;
3451 priv = this->private;
3452 par_readables = alloca0(priv->child_count);
3453 success = alloca0(priv->child_count);
3454
3455 ret = afr_inode_read_subvol_get(local->loc.parent, this, par_readables,
3456 NULL, NULL);
3457 if (ret < 0 || AFR_COUNT(par_readables, priv->child_count) == 0) {
3458 /* In this case set par_readables to all 1 so that name_heal
3459 * need checks at the end of this function will flag missing
3460 * entry when name state mismatches*/
3461 memset(par_readables, 1, priv->child_count);
3462 }
3463
3464 for (i = 0; i < priv->child_count; i++) {
3465 if (!replies[i].valid)
3466 continue;
3467
3468 if (replies[i].op_ret == 0) {
3469 if (gf_uuid_is_null(gfid)) {
3470 gf_uuid_copy(gfid, replies[i].poststat.ia_gfid);
3471 }
3472 success[i] = 1;
3473 } else {
3474 if ((replies[i].op_errno != ENOTCONN) &&
3475 (replies[i].op_errno != ENOENT) &&
3476 (replies[i].op_errno != ESTALE)) {
3477 op_errno = replies[i].op_errno;
3478 }
3479 }
3480
3481 /*gfid is missing, needs heal*/
3482 if ((replies[i].op_ret == -1) && (replies[i].op_errno == ENODATA)) {
3483 goto name_heal;
3484 }
3485
3486 if (first == -1) {
3487 first = i;
3488 continue;
3489 }
3490
3491 if (replies[i].op_ret != replies[first].op_ret) {
3492 name_state_mismatch = _gf_true;
3493 }
3494
3495 if (replies[i].op_ret == 0) {
3496 /* Rename after this lookup may succeed if we don't do
3497 * a name-heal and the destination may not have pending xattrs
3498 * to indicate which name is good and which is bad so always do
3499 * this heal*/
3500 if (gf_uuid_compare(replies[i].poststat.ia_gfid, gfid)) {
3501 goto name_heal;
3502 }
3503 }
3504 }
3505
3506 if (name_state_mismatch) {
3507 if (!priv->quorum_count)
3508 goto name_heal;
3509 if (!afr_has_quorum(success, priv, NULL))
3510 goto name_heal;
3511 if (op_errno)
3512 goto name_heal;
3513 for (i = 0; i < priv->child_count; i++) {
3514 if (!replies[i].valid)
3515 continue;
3516 if (par_readables[i] && replies[i].op_ret < 0 &&
3517 replies[i].op_errno != ENOTCONN) {
3518 goto name_heal;
3519 }
3520 }
3521 }
3522
3523 goto metadata_heal;
3524
3525 name_heal:
3526 heal = afr_frame_create(this, NULL);
3527 if (!heal)
3528 goto metadata_heal;
3529
3530 ret = synctask_new(this->ctx->env, afr_lookup_selfheal_wrap,
3531 afr_refresh_selfheal_done, heal, frame);
3532 if (ret) {
3533 AFR_STACK_DESTROY(heal);
3534 goto metadata_heal;
3535 }
3536 return ret;
3537
3538 metadata_heal:
3539 ret = afr_lookup_metadata_heal_check(frame, this);
3540
3541 return ret;
3542 }
3543
3544 int
3545 afr_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
3546 int op_errno, inode_t *inode, struct iatt *buf, dict_t *xdata,
3547 struct iatt *postparent)
3548 {
3549 afr_local_t *local = NULL;
3550 int call_count = -1;
3551 int child_index = -1;
3552 GF_UNUSED int ret = 0;
3553 int8_t need_heal = 1;
3554
3555 child_index = (long)cookie;
3556
3557 local = frame->local;
3558
3559 local->replies[child_index].valid = 1;
3560 local->replies[child_index].op_ret = op_ret;
3561 local->replies[child_index].op_errno = op_errno;
3562 /*
3563 * On revalidate lookup if the gfid-changed, afr should unwind the fop
3564 * with ESTALE so that a fresh lookup will be sent by the top xlator.
3565 * So remember it.
3566 */
3567 if (xdata && dict_get_sizen(xdata, "gfid-changed"))
3568 local->cont.lookup.needs_fresh_lookup = _gf_true;
3569
3570 if (xdata) {
3571 ret = dict_get_int8(xdata, "link-count", &need_heal);
3572 local->replies[child_index].need_heal = need_heal;
3573 } else {
3574 local->replies[child_index].need_heal = need_heal;
3575 }
3576 if (op_ret != -1) {
3577 local->replies[child_index].poststat = *buf;
3578 local->replies[child_index].postparent = *postparent;
3579 if (xdata)
3580 local->replies[child_index].xdata = dict_ref(xdata);
3581 }
3582
3583 call_count = afr_frame_return(frame);
3584 if (call_count == 0) {
3585 afr_set_need_heal(this, local);
3586 afr_lookup_entry_heal(frame, this);
3587 }
3588
3589 return 0;
3590 }
3591
3592 static void
3593 afr_discover_unwind(call_frame_t *frame, xlator_t *this)
3594 {
3595 afr_private_t *priv = NULL;
3596 afr_local_t *local = NULL;
3597 int read_subvol = -1;
3598 int ret = 0;
3599 unsigned char *data_readable = NULL;
3600 unsigned char *success_replies = NULL;
3601
3602 priv = this->private;
3603 local = frame->local;
3604 data_readable = alloca0(priv->child_count);
3605 success_replies = alloca0(priv->child_count);
3606
3607 afr_fill_success_replies(local, priv, success_replies);
3608 if (AFR_COUNT(success_replies, priv->child_count) > 0)
3609 local->op_ret = 0;
3610
3611 if (local->op_ret < 0) {
3612 local->op_ret = -1;
3613 local->op_errno = afr_final_errno(frame->local, this->private);
3614 goto error;
3615 }
3616
3617 if (!afr_has_quorum(success_replies, priv, frame))
3618 goto unwind;
3619
3620 ret = afr_replies_interpret(local, this, local->inode, NULL);
3621 if (ret) {
3622 afr_inode_need_refresh_set(local->inode, this);
3623 }
3624
3625 read_subvol = afr_read_subvol_decide(local->inode, this, NULL,
3626 data_readable);
3627
3628 unwind:
3629 afr_attempt_readsubvol_set(frame, this, success_replies, data_readable,
3630 &read_subvol);
3631 if (read_subvol == -1)
3632 goto error;
3633
3634 if (AFR_IS_ARBITER_BRICK(priv, read_subvol) && local->op_ret == 0) {
3635 local->op_ret = -1;
3636 local->op_errno = ENOTCONN;
3637 gf_msg_debug(this->name, 0,
3638 "Arbiter cannot be a read subvol "
3639 "for %s",
3640 local->loc.path);
3641 }
3642
3643 AFR_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
3644 local->inode, &local->replies[read_subvol].poststat,
3645 local->replies[read_subvol].xdata,
3646 &local->replies[read_subvol].postparent);
3647 return;
3648
3649 error:
3650 AFR_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno, NULL, NULL,
3651 NULL, NULL);
3652 }
3653
3654 static int
3655 afr_ta_id_file_check(void *opaque)
3656 {
3657 afr_private_t *priv = NULL;
3658 xlator_t *this = NULL;
3659 loc_t loc = {
3660 0,
3661 };
3662 struct iatt stbuf = {
3663 0,
3664 };
3665 dict_t *dict = NULL;
3666 uuid_t gfid = {
3667 0,
3668 };
3669 fd_t *fd = NULL;
3670 int ret = 0;
3671
3672 this = opaque;
3673 priv = this->private;
3674
3675 ret = afr_fill_ta_loc(this, &loc, _gf_false);
3676 if (ret) {
3677 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
3678 "Failed to populate thin-arbiter loc for: %s.", loc.name);
3679 goto out;
3680 }
3681
3682 ret = syncop_lookup(priv->children[THIN_ARBITER_BRICK_INDEX], &loc, &stbuf,
3683 0, 0, 0);
3684 if (ret == 0) {
3685 goto out;
3686 } else if (ret == -ENOENT) {
3687 fd = fd_create(loc.inode, getpid());
3688 if (!fd)
3689 goto out;
3690 dict = dict_new();
3691 if (!dict)
3692 goto out;
3693 gf_uuid_generate(gfid);
3694 ret = dict_set_gfuuid(dict, "gfid-req", gfid, true);
3695 ret = syncop_create(priv->children[THIN_ARBITER_BRICK_INDEX], &loc,
3696 O_RDWR, 0664, fd, &stbuf, dict, NULL);
3697 }
3698
3699 out:
3700 if (ret == 0) {
3701 gf_uuid_copy(priv->ta_gfid, stbuf.ia_gfid);
3702 } else {
3703 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
3704 "Failed to lookup/create thin-arbiter id file.");
3705 }
3706 if (dict)
3707 dict_unref(dict);
3708 if (fd)
3709 fd_unref(fd);
3710 loc_wipe(&loc);
3711
3712 return 0;
3713 }
3714
3715 static int
3716 afr_ta_id_file_check_cbk(int ret, call_frame_t *ta_frame, void *opaque)
3717 {
3718 return 0;
3719 }
3720
3721 static void
3722 afr_discover_done(call_frame_t *frame, xlator_t *this)
3723 {
3724 int ret = 0;
3725 afr_private_t *priv = NULL;
3726
3727 priv = this->private;
3728 if (!priv->thin_arbiter_count)
3729 goto unwind;
3730 if (!gf_uuid_is_null(priv->ta_gfid))
3731 goto unwind;
3732
3733 ret = synctask_new(this->ctx->env, afr_ta_id_file_check,
3734 afr_ta_id_file_check_cbk, NULL, this);
3735 if (ret)
3736 goto unwind;
3737 unwind:
3738 afr_discover_unwind(frame, this);
3739 }
3740
3741 int
3742 afr_discover_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
3743 int op_errno, inode_t *inode, struct iatt *buf, dict_t *xdata,
3744 struct iatt *postparent)
3745 {
3746 afr_local_t *local = NULL;
3747 int call_count = -1;
3748 int child_index = -1;
3749 GF_UNUSED int ret = 0;
3750 int8_t need_heal = 1;
3751
3752 child_index = (long)cookie;
3753
3754 local = frame->local;
3755
3756 local->replies[child_index].valid = 1;
3757 local->replies[child_index].op_ret = op_ret;
3758 local->replies[child_index].op_errno = op_errno;
3759 if (op_ret != -1) {
3760 local->replies[child_index].poststat = *buf;
3761 local->replies[child_index].postparent = *postparent;
3762 if (xdata)
3763 local->replies[child_index].xdata = dict_ref(xdata);
3764 }
3765
3766 if (local->do_discovery && (op_ret == 0))
3767 afr_attempt_local_discovery(this, child_index);
3768
3769 if (xdata) {
3770 ret = dict_get_int8(xdata, "link-count", &need_heal);
3771 local->replies[child_index].need_heal = need_heal;
3772 } else {
3773 local->replies[child_index].need_heal = need_heal;
3774 }
3775
3776 call_count = afr_frame_return(frame);
3777 if (call_count == 0) {
3778 afr_set_need_heal(this, local);
3779 afr_lookup_metadata_heal_check(frame, this);
3780 }
3781
3782 return 0;
3783 }
3784
3785 int
3786 afr_discover_do(call_frame_t *frame, xlator_t *this, int err)
3787 {
3788 int ret = 0;
3789 int i = 0;
3790 afr_local_t *local = NULL;
3791 afr_private_t *priv = NULL;
3792 int call_count = 0;
3793
3794 local = frame->local;
3795 priv = this->private;
3796
3797 if (err) {
3798 local->op_errno = err;
3799 goto out;
3800 }
3801
3802 call_count = local->call_count = AFR_COUNT(local->child_up,
3803 priv->child_count);
3804
3805 ret = afr_lookup_xattr_req_prepare(local, this, local->xattr_req,
3806 &local->loc);
3807 if (ret) {
3808 local->op_errno = -ret;
3809 goto out;
3810 }
3811
3812 for (i = 0; i < priv->child_count; i++) {
3813 if (local->child_up[i]) {
3814 STACK_WIND_COOKIE(
3815 frame, afr_discover_cbk, (void *)(long)i, priv->children[i],
3816 priv->children[i]->fops->lookup, &local->loc, local->xattr_req);
3817 if (!--call_count)
3818 break;
3819 }
3820 }
3821
3822 return 0;
3823 out:
3824 AFR_STACK_UNWIND(lookup, frame, -1, local->op_errno, 0, 0, 0, 0);
3825 return 0;
3826 }
3827
3828 int
3829 afr_discover(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
3830 {
3831 int op_errno = ENOMEM;
3832 afr_private_t *priv = NULL;
3833 afr_local_t *local = NULL;
3834 int event = 0;
3835
3836 priv = this->private;
3837
3838 local = AFR_FRAME_INIT(frame, op_errno);
3839 if (!local)
3840 goto out;
3841
3842 if (!local->call_count) {
3843 op_errno = ENOTCONN;
3844 goto out;
3845 }
3846
3847 if (__is_root_gfid(loc->inode->gfid)) {
3848 if (priv->choose_local && !priv->did_discovery) {
3849 /* Logic to detect which subvolumes of AFR are
3850 local, in order to prefer them for reads
3851 */
3852 local->do_discovery = _gf_true;
3853 priv->did_discovery = _gf_true;
3854 }
3855 }
3856
3857 local->op = GF_FOP_LOOKUP;
3858
3859 loc_copy(&local->loc, loc);
3860
3861 local->inode = inode_ref(loc->inode);
3862
3863 if (xattr_req) {
3864 /* If xattr_req was null, afr_lookup_xattr_req_prepare() will
3865 allocate one for us */
3866 local->xattr_req = dict_copy_with_ref(xattr_req, NULL);
3867 if (!local->xattr_req) {
3868 op_errno = ENOMEM;
3869 goto out;
3870 }
3871 }
3872
3873 if (gf_uuid_is_null(loc->inode->gfid)) {
3874 afr_discover_do(frame, this, 0);
3875 return 0;
3876 }
3877
3878 afr_read_subvol_get(loc->inode, this, NULL, NULL, &event,
3879 AFR_DATA_TRANSACTION, NULL);
3880
3881 afr_discover_do(frame, this, 0);
3882
3883 return 0;
3884 out:
3885 AFR_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
3886 return 0;
3887 }
3888
3889 static int
3890 afr_lookup_do(call_frame_t *frame, xlator_t *this, int err)
3891 {
3892 int ret = 0;
3893 int i = 0;
3894 afr_local_t *local = NULL;
3895 afr_private_t *priv = NULL;
3896 int call_count = 0;
3897
3898 local = frame->local;
3899 priv = this->private;
3900
3901 if (err < 0) {
3902 local->op_errno = err;
3903 goto out;
3904 }
3905
3906 call_count = local->call_count = AFR_COUNT(local->child_up,
3907 priv->child_count);
3908
3909 ret = afr_lookup_xattr_req_prepare(local, this, local->xattr_req,
3910 &local->loc);
3911 if (ret) {
3912 local->op_errno = -ret;
3913 goto out;
3914 }
3915
3916 for (i = 0; i < priv->child_count; i++) {
3917 if (local->child_up[i]) {
3918 STACK_WIND_COOKIE(
3919 frame, afr_lookup_cbk, (void *)(long)i, priv->children[i],
3920 priv->children[i]->fops->lookup, &local->loc, local->xattr_req);
3921 if (!--call_count)
3922 break;
3923 }
3924 }
3925 return 0;
3926 out:
3927 AFR_STACK_UNWIND(lookup, frame, -1, local->op_errno, 0, 0, 0, 0);
3928 return 0;
3929 }
3930
3931 /*
3932 * afr_lookup()
3933 *
3934 * The goal here is to figure out what the element getting looked up is.
3935 * i.e what is the GFID, inode type and a conservative estimate of the
3936 * inode attributes are.
3937 *
3938 * As we lookup, operations may be underway on the entry name and the
3939 * inode. In lookup() we are primarily concerned only with the entry
3940 * operations. If the entry is getting unlinked or renamed, we detect
3941 * what operation is underway by querying for on-going transactions and
3942 * pending self-healing on the entry through xdata.
3943 *
3944 * If the entry is a file/dir, it may need self-heal and/or in a
3945 * split-brain condition. Lookup is not the place to worry about these
3946 * conditions. Outcast marking will naturally handle them in the read
3947 * paths.
3948 *
3949 * Here is a brief goal of what we are trying to achieve:
3950 *
3951 * - LOOKUP on all subvolumes concurrently, querying on-going transaction
3952 * and pending self-heal info from the servers.
3953 *
3954 * - If all servers reply the same inode type and GFID, the overall call
3955 * MUST be a success.
3956 *
3957 * - If inode types or GFIDs mismatch, and there IS either an on-going
3958 * transaction or pending self-heal, inspect what the nature of the
3959 * transaction or pending heal is, and select the appropriate subvolume's
3960 * reply as the winner.
3961 *
3962 * - If inode types or GFIDs mismatch, and there are no on-going transactions
3963 * or pending self-heal on the entry name on any of the servers, fail the
3964 * lookup with EIO. Something has gone wrong beyond reasonable action.
3965 */
3966
3967 int
3968 afr_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
3969 {
3970 afr_local_t *local = NULL;
3971 int32_t op_errno = 0;
3972 int event = 0;
3973 int ret = 0;
3974
3975 if (loc_is_nameless(loc)) {
3976 if (xattr_req)
3977 dict_del_sizen(xattr_req, "gfid-req");
3978 afr_discover(frame, this, loc, xattr_req);
3979 return 0;
3980 }
3981
3982 if (__is_root_gfid(loc->parent->gfid) &&
3983 afr_is_private_directory(this->private, loc->name, frame->root->pid)) {
3984 op_errno = EPERM;
3985 goto out;
3986 }
3987
3988 local = AFR_FRAME_INIT(frame, op_errno);
3989 if (!local)
3990 goto out;
3991
3992 if (!local->call_count) {
3993 op_errno = ENOTCONN;
3994 goto out;
3995 }
3996
3997 local->op = GF_FOP_LOOKUP;
3998
3999 loc_copy(&local->loc, loc);
4000
4001 local->inode = inode_ref(loc->inode);
4002
4003 if (xattr_req) {
4004 /* If xattr_req was null, afr_lookup_xattr_req_prepare() will
4005 allocate one for us */
4006 local->xattr_req = dict_copy_with_ref(xattr_req, NULL);
4007 if (!local->xattr_req) {
4008 op_errno = ENOMEM;
4009 goto out;
4010 }
4011 ret = dict_get_gfuuid(local->xattr_req, "gfid-req",
4012 &local->cont.lookup.gfid_req);
4013 if (ret == 0) {
4014 dict_del_sizen(local->xattr_req, "gfid-req");
4015 }
4016 }
4017
4018 afr_read_subvol_get(loc->parent, this, NULL, NULL, &event,
4019 AFR_DATA_TRANSACTION, NULL);
4020
4021 afr_lookup_do(frame, this, 0);
4022
4023 return 0;
4024 out:
4025 AFR_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
4026
4027 return 0;
4028 }
4029
4030 static void
4031 _afr_cleanup_fd_ctx(xlator_t *this, afr_fd_ctx_t *fd_ctx)
4032 {
4033 afr_private_t *priv = this->private;
4034
4035 if (fd_ctx->lk_heal_info) {
4036 LOCK(&priv->lock);
4037 {
4038 list_del(&fd_ctx->lk_heal_info->pos);
4039 }
4040 UNLOCK(&priv->lock);
4041 afr_lk_heal_info_cleanup(fd_ctx->lk_heal_info);
4042 fd_ctx->lk_heal_info = NULL;
4043 }
4044 GF_FREE(fd_ctx->opened_on);
4045 GF_FREE(fd_ctx);
4046 return;
4047 }
4048
4049 void
4050 afr_cleanup_fd_ctx(xlator_t *this, fd_t *fd)
4051 {
4052 afr_fd_ctx_t *fd_ctx = NULL;
4053
4054 fd_ctx = fd_ctx_get_ptr(fd, this);
4055 if (fd_ctx)
4056 _afr_cleanup_fd_ctx(this, fd_ctx);
4057 }
4058
4059 int
4060 afr_release(xlator_t *this, fd_t *fd)
4061 {
4062 afr_cleanup_fd_ctx(this, fd);
4063
4064 return 0;
4065 }
4066
4067 static afr_fd_ctx_t *
4068 __afr_fd_ctx_set(xlator_t *this, fd_t *fd)
4069 {
4070 afr_private_t *priv = NULL;
4071 int ret = -1;
4072 afr_fd_ctx_t *fd_ctx = NULL;
4073 int i = 0;
4074
4075 VALIDATE_OR_GOTO(this->private, out);
4076
4077 priv = this->private;
4078
4079 fd_ctx = GF_CALLOC(1, sizeof(afr_fd_ctx_t), gf_afr_mt_afr_fd_ctx_t);
4080 if (!fd_ctx) {
4081 ret = -ENOMEM;
4082 goto out;
4083 }
4084
4085 fd_ctx->opened_on = GF_CALLOC(sizeof(*fd_ctx->opened_on), priv->child_count,
4086 gf_afr_mt_int32_t);
4087 if (!fd_ctx->opened_on) {
4088 ret = -ENOMEM;
4089 goto out;
4090 }
4091
4092 for (i = 0; i < priv->child_count; i++) {
4093 if (fd_is_anonymous(fd))
4094 fd_ctx->opened_on[i] = AFR_FD_OPENED;
4095 else
4096 fd_ctx->opened_on[i] = AFR_FD_NOT_OPENED;
4097 }
4098
4099 fd_ctx->readdir_subvol = -1;
4100 fd_ctx->lk_heal_info = NULL;
4101
4102 ret = __fd_ctx_set(fd, this, (uint64_t)(long)fd_ctx);
4103 if (ret)
4104 gf_msg_debug(this->name, 0, "failed to set fd ctx (%p)", fd);
4105 out:
4106 if (ret && fd_ctx) {
4107 _afr_cleanup_fd_ctx(this, fd_ctx);
4108 fd_ctx = NULL;
4109 }
4110 return fd_ctx;
4111 }
4112
4113 static afr_fd_ctx_t *
4114 __afr_fd_ctx_get(fd_t *fd, xlator_t *this)
4115 {
4116 afr_fd_ctx_t *fd_ctx = NULL;
4117
4118 fd_ctx = __fd_ctx_get_ptr(fd, this);
4119
4120 if (!fd_ctx) {
4121 fd_ctx = __afr_fd_ctx_set(this, fd);
4122 }
4123 return fd_ctx;
4124 }
4125
4126 afr_fd_ctx_t *
4127 afr_fd_ctx_get(fd_t *fd, xlator_t *this)
4128 {
4129 afr_fd_ctx_t *fd_ctx = NULL;
4130
4131 LOCK(&fd->lock);
4132 {
4133 fd_ctx = __afr_fd_ctx_get(fd, this);
4134 }
4135 UNLOCK(&fd->lock);
4136
4137 return fd_ctx;
4138 }
4139
4140 /* {{{ flush */
4141
4142 int
4143 afr_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
4144 int32_t op_errno, dict_t *xdata)
4145 {
4146 afr_local_t *local = NULL;
4147 int call_count = -1;
4148
4149 local = frame->local;
4150
4151 LOCK(&frame->lock);
4152 {
4153 if (op_ret != -1) {
4154 local->op_ret = op_ret;
4155 if (!local->xdata_rsp && xdata)
4156 local->xdata_rsp = dict_ref(xdata);
4157 } else {
4158 local->op_errno = op_errno;
4159 }
4160 call_count = --local->call_count;
4161 }
4162 UNLOCK(&frame->lock);
4163
4164 if (call_count == 0)
4165 AFR_STACK_UNWIND(flush, frame, local->op_ret, local->op_errno,
4166 local->xdata_rsp);
4167
4168 return 0;
4169 }
4170
4171 static int
4172 afr_flush_wrapper(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
4173 {
4174 int i = 0;
4175 afr_local_t *local = NULL;
4176 afr_private_t *priv = NULL;
4177 int call_count = -1;
4178
4179 priv = this->private;
4180 local = frame->local;
4181 call_count = local->call_count;
4182
4183 for (i = 0; i < priv->child_count; i++) {
4184 if (local->child_up[i]) {
4185 STACK_WIND_COOKIE(frame, afr_flush_cbk, (void *)(long)i,
4186 priv->children[i], priv->children[i]->fops->flush,
4187 local->fd, xdata);
4188 if (!--call_count)
4189 break;
4190 }
4191 }
4192
4193 return 0;
4194 }
4195
4196 static afr_local_t *
4197 afr_wakeup_same_fd_delayed_op(xlator_t *this, afr_lock_t *lock, fd_t *fd)
4198 {
4199 afr_local_t *local = NULL;
4200
4201 if (lock->delay_timer) {
4202 local = list_entry(lock->post_op.next, afr_local_t,
4203 transaction.owner_list);
4204 if (fd == local->fd) {
4205 if (gf_timer_call_cancel(this->ctx, lock->delay_timer)) {
4206 local = NULL;
4207 } else {
4208 lock->release = _gf_true;
4209 lock->delay_timer = NULL;
4210 }
4211 } else {
4212 local = NULL;
4213 }
4214 }
4215
4216 return local;
4217 }
4218
4219 static void
4220 afr_delayed_changelog_wake_resume(xlator_t *this, inode_t *inode,
4221 call_stub_t *stub)
4222 {
4223 afr_inode_ctx_t *ctx = NULL;
4224 afr_lock_t *lock = NULL;
4225 afr_local_t *metadata_local = NULL;
4226 afr_local_t *data_local = NULL;
4227 LOCK(&inode->lock);
4228 {
4229 ctx = __afr_inode_ctx_get(this, inode);
4230 if (ctx == NULL)
4231 goto unlock;
4232 lock = &ctx->lock[AFR_DATA_TRANSACTION];
4233 data_local = afr_wakeup_same_fd_delayed_op(this, lock, stub->args.fd);
4234 lock = &ctx->lock[AFR_METADATA_TRANSACTION];
4235 metadata_local = afr_wakeup_same_fd_delayed_op(this, lock,
4236 stub->args.fd);
4237 }
4238 unlock:
4239 UNLOCK(&inode->lock);
4240
4241 if (data_local) {
4242 data_local->transaction.resume_stub = stub;
4243 } else if (metadata_local) {
4244 metadata_local->transaction.resume_stub = stub;
4245 } else {
4246 call_resume(stub);
4247 }
4248 if (data_local) {
4249 afr_delayed_changelog_wake_up_cbk(data_local);
4250 }
4251 if (metadata_local) {
4252 afr_delayed_changelog_wake_up_cbk(metadata_local);
4253 }
4254 }
4255
4256 int
4257 afr_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
4258 {
4259 afr_local_t *local = NULL;
4260 call_stub_t *stub = NULL;
4261 int op_errno = ENOMEM;
4262
4263 AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
4264 local = AFR_FRAME_INIT(frame, op_errno);
4265 if (!local)
4266 goto out;
4267
4268 local->op = GF_FOP_FLUSH;
4269 if (!afr_is_consistent_io_possible(local, this->private, &op_errno))
4270 goto out;
4271
4272 local->fd = fd_ref(fd);
4273
4274 stub = fop_flush_stub(frame, afr_flush_wrapper, fd, xdata);
4275 if (!stub)
4276 goto out;
4277
4278 afr_delayed_changelog_wake_resume(this, fd->inode, stub);
4279
4280 return 0;
4281 out:
4282 AFR_STACK_UNWIND(flush, frame, -1, op_errno, NULL);
4283 return 0;
4284 }
4285
4286 int
4287 afr_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
4288 int32_t op_ret, int32_t op_errno, dict_t *xdata)
4289 {
4290 afr_local_t *local = NULL;
4291 int call_count = -1;
4292
4293 local = frame->local;
4294
4295 LOCK(&frame->lock);
4296 {
4297 if (op_ret == 0) {
4298 local->op_ret = 0;
4299 if (!local->xdata_rsp && xdata)
4300 local->xdata_rsp = dict_ref(xdata);
4301 } else {
4302 local->op_errno = op_errno;
4303 }
4304 call_count = --local->call_count;
4305 }
4306 UNLOCK(&frame->lock);
4307
4308 if (call_count == 0)
4309 AFR_STACK_UNWIND(fsyncdir, frame, local->op_ret, local->op_errno,
4310 local->xdata_rsp);
4311
4312 return 0;
4313 }
4314
4315 int
4316 afr_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
4317 dict_t *xdata)
4318 {
4319 afr_private_t *priv = NULL;
4320 afr_local_t *local = NULL;
4321 int i = 0;
4322 int32_t call_count = 0;
4323 int32_t op_errno = ENOMEM;
4324
4325 priv = this->private;
4326
4327 local = AFR_FRAME_INIT(frame, op_errno);
4328 if (!local)
4329 goto out;
4330
4331 local->op = GF_FOP_FSYNCDIR;
4332 if (!afr_is_consistent_io_possible(local, priv, &op_errno))
4333 goto out;
4334
4335 call_count = local->call_count;
4336 for (i = 0; i < priv->child_count; i++) {
4337 if (local->child_up[i]) {
4338 STACK_WIND(frame, afr_fsyncdir_cbk, priv->children[i],
4339 priv->children[i]->fops->fsyncdir, fd, datasync, xdata);
4340 if (!--call_count)
4341 break;
4342 }
4343 }
4344
4345 return 0;
4346 out:
4347 AFR_STACK_UNWIND(fsyncdir, frame, -1, op_errno, NULL);
4348
4349 return 0;
4350 }
4351
4352 /* }}} */
4353
4354 static int
4355 afr_serialized_lock_wind(call_frame_t *frame, xlator_t *this);
4356
4357 static gf_boolean_t
4358 afr_is_conflicting_lock_present(int32_t op_ret, int32_t op_errno)
4359 {
4360 if (op_ret == -1 && op_errno == EAGAIN)
4361 return _gf_true;
4362 return _gf_false;
4363 }
4364
4365 static void
4366 afr_fop_lock_unwind(call_frame_t *frame, glusterfs_fop_t op, int32_t op_ret,
4367 int32_t op_errno, dict_t *xdata)
4368 {
4369 switch (op) {
4370 case GF_FOP_INODELK:
4371 AFR_STACK_UNWIND(inodelk, frame, op_ret, op_errno, xdata);
4372 break;
4373 case GF_FOP_FINODELK:
4374 AFR_STACK_UNWIND(finodelk, frame, op_ret, op_errno, xdata);
4375 break;
4376 case GF_FOP_ENTRYLK:
4377 AFR_STACK_UNWIND(entrylk, frame, op_ret, op_errno, xdata);
4378 break;
4379 case GF_FOP_FENTRYLK:
4380 AFR_STACK_UNWIND(fentrylk, frame, op_ret, op_errno, xdata);
4381 break;
4382 default:
4383 break;
4384 }
4385 }
4386
4387 static void
4388 afr_fop_lock_wind(call_frame_t *frame, xlator_t *this, int child_index,
4389 int32_t (*lock_cbk)(call_frame_t *, void *, xlator_t *,
4390 int32_t, int32_t, dict_t *))
4391 {
4392 afr_local_t *local = frame->local;
4393 afr_private_t *priv = this->private;
4394 int i = child_index;
4395
4396 switch (local->op) {
4397 case GF_FOP_INODELK:
4398 STACK_WIND_COOKIE(
4399 frame, lock_cbk, (void *)(long)i, priv->children[i],
4400 priv->children[i]->fops->inodelk,
4401 (const char *)local->cont.inodelk.volume, &local->loc,
4402 local->cont.inodelk.cmd, &local->cont.inodelk.flock,
4403 local->cont.inodelk.xdata);
4404 break;
4405 case GF_FOP_FINODELK:
4406 STACK_WIND_COOKIE(
4407 frame, lock_cbk, (void *)(long)i, priv->children[i],
4408 priv->children[i]->fops->finodelk,
4409 (const char *)local->cont.inodelk.volume, local->fd,
4410 local->cont.inodelk.cmd, &local->cont.inodelk.flock,
4411 local->cont.inodelk.xdata);
4412 break;
4413 case GF_FOP_ENTRYLK:
4414 STACK_WIND_COOKIE(
4415 frame, lock_cbk, (void *)(long)i, priv->children[i],
4416 priv->children[i]->fops->entrylk, local->cont.entrylk.volume,
4417 &local->loc, local->cont.entrylk.basename,
4418 local->cont.entrylk.cmd, local->cont.entrylk.type,
4419 local->cont.entrylk.xdata);
4420 break;
4421 case GF_FOP_FENTRYLK:
4422 STACK_WIND_COOKIE(
4423 frame, lock_cbk, (void *)(long)i, priv->children[i],
4424 priv->children[i]->fops->fentrylk, local->cont.entrylk.volume,
4425 local->fd, local->cont.entrylk.basename,
4426 local->cont.entrylk.cmd, local->cont.entrylk.type,
4427 local->cont.entrylk.xdata);
4428 break;
4429 default:
4430 break;
4431 }
4432 }
4433
4434 static void
4435 afr_fop_lock_proceed(call_frame_t *frame)
4436 {
4437 afr_local_t *local = NULL;
4438 afr_private_t *priv = NULL;
4439
4440 local = frame->local;
4441 priv = frame->this->private;
4442
4443 if (local->fop_lock_state != AFR_FOP_LOCK_PARALLEL) {
4444 afr_fop_lock_unwind(frame, local->op, local->op_ret, local->op_errno,
4445 local->xdata_rsp);
4446 return;
4447 }
4448 /* At least one child is up */
4449 /*
4450 * Non-blocking locks also need to be serialized. Otherwise there is
4451 * a chance that both the mounts which issued same non-blocking inodelk
4452 * may endup not acquiring the lock on any-brick.
4453 * Ex: Mount1 and Mount2
4454 * request for full length lock on file f1. Mount1 afr may acquire the
4455 * partial lock on brick-1 and may not acquire the lock on brick-2
4456 * because Mount2 already got the lock on brick-2, vice versa. Since
4457 * both the mounts only got partial locks, afr treats them as failure in
4458 * gaining the locks and unwinds with EAGAIN errno.
4459 */
4460 local->op_ret = -1;
4461 local->op_errno = EUCLEAN;
4462 local->fop_lock_state = AFR_FOP_LOCK_SERIAL;
4463 afr_local_replies_wipe(local, priv);
4464 if (local->xdata_rsp)
4465 dict_unref(local->xdata_rsp);
4466 local->xdata_rsp = NULL;
4467 switch (local->op) {
4468 case GF_FOP_INODELK:
4469 case GF_FOP_FINODELK:
4470 local->cont.inodelk.cmd = local->cont.inodelk.in_cmd;
4471 gf_flock_copy(&local->cont.inodelk.flock,
4472 &local->cont.inodelk.in_flock);
4473 if (local->cont.inodelk.xdata)
4474 dict_unref(local->cont.inodelk.xdata);
4475 local->cont.inodelk.xdata = NULL;
4476 if (local->xdata_req)
4477 local->cont.inodelk.xdata = dict_ref(local->xdata_req);
4478 break;
4479 case GF_FOP_ENTRYLK:
4480 case GF_FOP_FENTRYLK:
4481 local->cont.entrylk.cmd = local->cont.entrylk.in_cmd;
4482 if (local->cont.entrylk.xdata)
4483 dict_unref(local->cont.entrylk.xdata);
4484 local->cont.entrylk.xdata = NULL;
4485 if (local->xdata_req)
4486 local->cont.entrylk.xdata = dict_ref(local->xdata_req);
4487 break;
4488 default:
4489 break;
4490 }
4491 afr_serialized_lock_wind(frame, frame->this);
4492 }
4493
4494 static int32_t
4495 afr_unlock_partial_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
4496 int32_t op_ret, int32_t op_errno, dict_t *xdata)
4497
4498 {
4499 afr_local_t *local = NULL;
4500 afr_private_t *priv = NULL;
4501 int call_count = -1;
4502 int child_index = (long)cookie;
4503 uuid_t gfid = {0};
4504
4505 local = frame->local;
4506 priv = this->private;
4507
4508 if (op_ret < 0 && op_errno != ENOTCONN) {
4509 if (local->fd)
4510 gf_uuid_copy(gfid, local->fd->inode->gfid);
4511 else
4512 loc_gfid(&local->loc, gfid);
4513 gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_UNLOCK_FAIL,
4514 "%s: Failed to unlock %s on %s "
4515 "with lk_owner: %s",
4516 uuid_utoa(gfid), gf_fop_list[local->op],
4517 priv->children[child_index]->name,
4518 lkowner_utoa(&frame->root->lk_owner));
4519 }
4520
4521 call_count = afr_frame_return(frame);
4522 if (call_count == 0)
4523 afr_fop_lock_proceed(frame);
4524
4525 return 0;
4526 }
4527
4528 static int32_t
4529 afr_unlock_locks_and_proceed(call_frame_t *frame, xlator_t *this,
4530 int call_count)
4531 {
4532 int i = 0;
4533 afr_private_t *priv = NULL;
4534 afr_local_t *local = NULL;
4535
4536 if (call_count == 0) {
4537 afr_fop_lock_proceed(frame);
4538 goto out;
4539 }
4540
4541 local = frame->local;
4542 priv = this->private;
4543 local->call_count = call_count;
4544 switch (local->op) {
4545 case GF_FOP_INODELK:
4546 case GF_FOP_FINODELK:
4547 local->cont.inodelk.flock.l_type = F_UNLCK;
4548 local->cont.inodelk.cmd = F_SETLK;
4549 if (local->cont.inodelk.xdata)
4550 dict_unref(local->cont.inodelk.xdata);
4551 local->cont.inodelk.xdata = NULL;
4552 break;
4553 case GF_FOP_ENTRYLK:
4554 case GF_FOP_FENTRYLK:
4555 local->cont.entrylk.cmd = ENTRYLK_UNLOCK;
4556 if (local->cont.entrylk.xdata)
4557 dict_unref(local->cont.entrylk.xdata);
4558 local->cont.entrylk.xdata = NULL;
4559 break;
4560 default:
4561 break;
4562 }
4563
4564 for (i = 0; i < priv->child_count; i++) {
4565 if (!local->replies[i].valid)
4566 continue;
4567
4568 if (local->replies[i].op_ret == -1)
4569 continue;
4570
4571 afr_fop_lock_wind(frame, this, i, afr_unlock_partial_lock_cbk);
4572
4573 if (!--call_count)
4574 break;
4575 }
4576
4577 out:
4578 return 0;
4579 }
4580
4581 static int32_t
4582 afr_fop_lock_done(call_frame_t *frame, xlator_t *this)
4583 {
4584 int i = 0;
4585 int lock_count = 0;
4586 unsigned char *success = NULL;
4587
4588 afr_local_t *local = NULL;
4589 afr_private_t *priv = NULL;
4590
4591 local = frame->local;
4592 priv = this->private;
4593 success = alloca0(priv->child_count);
4594
4595 for (i = 0; i < priv->child_count; i++) {
4596 if (!local->replies[i].valid)
4597 continue;
4598
4599 if (local->replies[i].op_ret == 0) {
4600 lock_count++;
4601 success[i] = 1;
4602 }
4603
4604 if (local->op_ret == -1 && local->op_errno == EAGAIN)
4605 continue;
4606
4607 if ((local->replies[i].op_ret == -1) &&
4608 (local->replies[i].op_errno == EAGAIN)) {
4609 local->op_ret = -1;
4610 local->op_errno = EAGAIN;
4611 continue;
4612 }
4613
4614 if (local->replies[i].op_ret == 0)
4615 local->op_ret = 0;
4616
4617 local->op_errno = local->replies[i].op_errno;
4618 }
4619
4620 if (afr_fop_lock_is_unlock(frame))
4621 goto unwind;
4622
4623 if (afr_is_conflicting_lock_present(local->op_ret, local->op_errno)) {
4624 afr_unlock_locks_and_proceed(frame, this, lock_count);
4625 } else if (priv->quorum_count && !afr_has_quorum(success, priv, NULL)) {
4626 local->fop_lock_state = AFR_FOP_LOCK_QUORUM_FAILED;
4627 local->op_ret = -1;
4628 local->op_errno = afr_final_errno(local, priv);
4629 if (local->op_errno == 0)
4630 local->op_errno = afr_quorum_errno(priv);
4631 afr_unlock_locks_and_proceed(frame, this, lock_count);
4632 } else {
4633 goto unwind;
4634 }
4635
4636 return 0;
4637 unwind:
4638 afr_fop_lock_unwind(frame, local->op, local->op_ret, local->op_errno,
4639 local->xdata_rsp);
4640 return 0;
4641 }
4642
4643 static int
4644 afr_common_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
4645 int32_t op_ret, int32_t op_errno, dict_t *xdata)
4646 {
4647 afr_local_t *local = NULL;
4648 int child_index = (long)cookie;
4649
4650 local = frame->local;
4651
4652 local->replies[child_index].valid = 1;
4653 local->replies[child_index].op_ret = op_ret;
4654 local->replies[child_index].op_errno = op_errno;
4655 if (op_ret == 0 && xdata) {
4656 local->replies[child_index].xdata = dict_ref(xdata);
4657 LOCK(&frame->lock);
4658 {
4659 if (!local->xdata_rsp)
4660 local->xdata_rsp = dict_ref(xdata);
4661 }
4662 UNLOCK(&frame->lock);
4663 }
4664 return 0;
4665 }
4666
4667 static int32_t
4668 afr_serialized_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
4669 int32_t op_ret, int32_t op_errno, dict_t *xdata)
4670
4671 {
4672 afr_local_t *local = NULL;
4673 afr_private_t *priv = NULL;
4674 int child_index = (long)cookie;
4675 int next_child = 0;
4676
4677 local = frame->local;
4678 priv = this->private;
4679
4680 afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
4681
4682 for (next_child = child_index + 1; next_child < priv->child_count;
4683 next_child++) {
4684 if (local->child_up[next_child])
4685 break;
4686 }
4687
4688 if (afr_is_conflicting_lock_present(op_ret, op_errno) ||
4689 (next_child == priv->child_count)) {
4690 afr_fop_lock_done(frame, this);
4691 } else {
4692 afr_fop_lock_wind(frame, this, next_child, afr_serialized_lock_cbk);
4693 }
4694
4695 return 0;
4696 }
4697
4698 static int
4699 afr_serialized_lock_wind(call_frame_t *frame, xlator_t *this)
4700 {
4701 afr_private_t *priv = NULL;
4702 afr_local_t *local = NULL;
4703 int i = 0;
4704
4705 priv = this->private;
4706 local = frame->local;
4707
4708 for (i = 0; i < priv->child_count; i++) {
4709 if (local->child_up[i]) {
4710 afr_fop_lock_wind(frame, this, i, afr_serialized_lock_cbk);
4711 break;
4712 }
4713 }
4714 return 0;
4715 }
4716
4717 static int32_t
4718 afr_parallel_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
4719 int32_t op_ret, int32_t op_errno, dict_t *xdata)
4720
4721 {
4722 int call_count = 0;
4723
4724 afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
4725
4726 call_count = afr_frame_return(frame);
4727 if (call_count == 0)
4728 afr_fop_lock_done(frame, this);
4729
4730 return 0;
4731 }
4732
4733 static int
4734 afr_parallel_lock_wind(call_frame_t *frame, xlator_t *this)
4735 {
4736 afr_private_t *priv = NULL;
4737 afr_local_t *local = NULL;
4738 int call_count = 0;
4739 int i = 0;
4740
4741 priv = this->private;
4742 local = frame->local;
4743 call_count = local->call_count;
4744
4745 for (i = 0; i < priv->child_count; i++) {
4746 if (!local->child_up[i])
4747 continue;
4748 afr_fop_lock_wind(frame, this, i, afr_parallel_lock_cbk);
4749 if (!--call_count)
4750 break;
4751 }
4752 return 0;
4753 }
4754
4755 static int
4756 afr_fop_handle_lock(call_frame_t *frame, xlator_t *this)
4757 {
4758 afr_local_t *local = frame->local;
4759 int op_errno = 0;
4760
4761 if (!afr_fop_lock_is_unlock(frame)) {
4762 if (!afr_is_consistent_io_possible(local, this->private, &op_errno))
4763 goto out;
4764
4765 switch (local->op) {
4766 case GF_FOP_INODELK:
4767 case GF_FOP_FINODELK:
4768 local->cont.inodelk.cmd = F_SETLK;
4769 break;
4770 case GF_FOP_ENTRYLK:
4771 case GF_FOP_FENTRYLK:
4772 local->cont.entrylk.cmd = ENTRYLK_LOCK_NB;
4773 break;
4774 default:
4775 break;
4776 }
4777 }
4778
4779 if (local->xdata_req) {
4780 switch (local->op) {
4781 case GF_FOP_INODELK:
4782 case GF_FOP_FINODELK:
4783 local->cont.inodelk.xdata = dict_ref(local->xdata_req);
4784 break;
4785 case GF_FOP_ENTRYLK:
4786 case GF_FOP_FENTRYLK:
4787 local->cont.entrylk.xdata = dict_ref(local->xdata_req);
4788 break;
4789 default:
4790 break;
4791 }
4792 }
4793
4794 local->fop_lock_state = AFR_FOP_LOCK_PARALLEL;
4795 afr_parallel_lock_wind(frame, this);
4796 out:
4797 return -op_errno;
4798 }
4799
4800 static int32_t
4801 afr_handle_inodelk(call_frame_t *frame, xlator_t *this, glusterfs_fop_t fop,
4802 const char *volume, loc_t *loc, fd_t *fd, int32_t cmd,
4803 struct gf_flock *flock, dict_t *xdata)
4804 {
4805 afr_local_t *local = NULL;
4806 int32_t op_errno = ENOMEM;
4807
4808 local = AFR_FRAME_INIT(frame, op_errno);
4809 if (!local)
4810 goto out;
4811
4812 local->op = fop;
4813 if (loc)
4814 loc_copy(&local->loc, loc);
4815 if (fd && (flock->l_type != F_UNLCK)) {
4816 AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
4817 local->fd = fd_ref(fd);
4818 }
4819
4820 local->cont.inodelk.volume = gf_strdup(volume);
4821 if (!local->cont.inodelk.volume) {
4822 op_errno = ENOMEM;
4823 goto out;
4824 }
4825
4826 local->cont.inodelk.in_cmd = cmd;
4827 local->cont.inodelk.cmd = cmd;
4828 gf_flock_copy(&local->cont.inodelk.in_flock, flock);
4829 gf_flock_copy(&local->cont.inodelk.flock, flock);
4830 if (xdata)
4831 local->xdata_req = dict_ref(xdata);
4832
4833 op_errno = -afr_fop_handle_lock(frame, frame->this);
4834 if (op_errno)
4835 goto out;
4836 return 0;
4837 out:
4838 afr_fop_lock_unwind(frame, fop, -1, op_errno, NULL);
4839
4840 return 0;
4841 }
4842
4843 int32_t
4844 afr_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
4845 int32_t cmd, struct gf_flock *flock, dict_t *xdata)
4846 {
4847 afr_handle_inodelk(frame, this, GF_FOP_INODELK, volume, loc, NULL, cmd,
4848 flock, xdata);
4849 return 0;
4850 }
4851
4852 int32_t
4853 afr_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
4854 int32_t cmd, struct gf_flock *flock, dict_t *xdata)
4855 {
4856 afr_handle_inodelk(frame, this, GF_FOP_FINODELK, volume, NULL, fd, cmd,
4857 flock, xdata);
4858 return 0;
4859 }
4860
4861 static int
4862 afr_handle_entrylk(call_frame_t *frame, xlator_t *this, glusterfs_fop_t fop,
4863 const char *volume, loc_t *loc, fd_t *fd,
4864 const char *basename, entrylk_cmd cmd, entrylk_type type,
4865 dict_t *xdata)
4866 {
4867 afr_local_t *local = NULL;
4868 int32_t op_errno = ENOMEM;
4869
4870 local = AFR_FRAME_INIT(frame, op_errno);
4871 if (!local)
4872 goto out;
4873
4874 local->op = fop;
4875 if (loc)
4876 loc_copy(&local->loc, loc);
4877 if (fd && (cmd != ENTRYLK_UNLOCK)) {
4878 AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
4879 local->fd = fd_ref(fd);
4880 }
4881 local->cont.entrylk.cmd = cmd;
4882 local->cont.entrylk.in_cmd = cmd;
4883 local->cont.entrylk.type = type;
4884 local->cont.entrylk.volume = gf_strdup(volume);
4885 local->cont.entrylk.basename = gf_strdup(basename);
4886 if (!local->cont.entrylk.volume || !local->cont.entrylk.basename) {
4887 op_errno = ENOMEM;
4888 goto out;
4889 }
4890 if (xdata)
4891 local->xdata_req = dict_ref(xdata);
4892 op_errno = -afr_fop_handle_lock(frame, frame->this);
4893 if (op_errno)
4894 goto out;
4895
4896 return 0;
4897 out:
4898 afr_fop_lock_unwind(frame, fop, -1, op_errno, NULL);
4899 return 0;
4900 }
4901
4902 int
4903 afr_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
4904 const char *basename, entrylk_cmd cmd, entrylk_type type,
4905 dict_t *xdata)
4906 {
4907 afr_handle_entrylk(frame, this, GF_FOP_ENTRYLK, volume, loc, NULL, basename,
4908 cmd, type, xdata);
4909 return 0;
4910 }
4911
4912 int
4913 afr_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
4914 const char *basename, entrylk_cmd cmd, entrylk_type type,
4915 dict_t *xdata)
4916 {
4917 afr_handle_entrylk(frame, this, GF_FOP_FENTRYLK, volume, NULL, fd, basename,
4918 cmd, type, xdata);
4919 return 0;
4920 }
4921
4922 int
4923 afr_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
4924 int op_errno, struct statvfs *statvfs, dict_t *xdata)
4925 {
4926 afr_local_t *local = NULL;
4927 int call_count = 0;
4928 struct statvfs *buf = NULL;
4929
4930 local = frame->local;
4931
4932 LOCK(&frame->lock);
4933 {
4934 if (op_ret != 0) {
4935 local->op_errno = op_errno;
4936 goto unlock;
4937 }
4938
4939 local->op_ret = op_ret;
4940
4941 buf = &local->cont.statfs.buf;
4942 if (local->cont.statfs.buf_set) {
4943 if (statvfs->f_bavail < buf->f_bavail) {
4944 *buf = *statvfs;
4945 if (xdata) {
4946 if (local->xdata_rsp)
4947 dict_unref(local->xdata_rsp);
4948 local->xdata_rsp = dict_ref(xdata);
4949 }
4950 }
4951 } else {
4952 *buf = *statvfs;
4953 local->cont.statfs.buf_set = 1;
4954 if (xdata)
4955 local->xdata_rsp = dict_ref(xdata);
4956 }
4957 }
4958 unlock:
4959 call_count = --local->call_count;
4960 UNLOCK(&frame->lock);
4961
4962 if (call_count == 0)
4963 AFR_STACK_UNWIND(statfs, frame, local->op_ret, local->op_errno,
4964 &local->cont.statfs.buf, local->xdata_rsp);
4965
4966 return 0;
4967 }
4968
4969 int
4970 afr_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
4971 {
4972 afr_local_t *local = NULL;
4973 afr_private_t *priv = NULL;
4974 int i = 0;
4975 int call_count = 0;
4976 int32_t op_errno = ENOMEM;
4977
4978 priv = this->private;
4979
4980 local = AFR_FRAME_INIT(frame, op_errno);
4981 if (!local)
4982 goto out;
4983
4984 local->op = GF_FOP_STATFS;
4985 if (!afr_is_consistent_io_possible(local, priv, &op_errno))
4986 goto out;
4987
4988 if (priv->arbiter_count == 1 && local->child_up[ARBITER_BRICK_INDEX])
4989 local->call_count--;
4990 call_count = local->call_count;
4991 if (!call_count) {
4992 op_errno = ENOTCONN;
4993 goto out;
4994 }
4995
4996 for (i = 0; i < priv->child_count; i++) {
4997 if (local->child_up[i]) {
4998 if (AFR_IS_ARBITER_BRICK(priv, i))
4999 continue;
5000 STACK_WIND(frame, afr_statfs_cbk, priv->children[i],
5001 priv->children[i]->fops->statfs, loc, xdata);
5002 if (!--call_count)
5003 break;
5004 }
5005 }
5006
5007 return 0;
5008 out:
5009 AFR_STACK_UNWIND(statfs, frame, -1, op_errno, NULL, NULL);
5010
5011 return 0;
5012 }
5013
5014 int32_t
5015 afr_lk_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
5016 int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
5017 dict_t *xdata)
5018 {
5019 afr_local_t *local = NULL;
5020 afr_private_t *priv = this->private;
5021 int call_count = -1;
5022 int child_index = (long)cookie;
5023
5024 local = frame->local;
5025
5026 if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
5027 gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_UNLOCK_FAIL,
5028 "gfid=%s: unlock failed on subvolume %s "
5029 "with lock owner %s",
5030 uuid_utoa(local->fd->inode->gfid),
5031 priv->children[child_index]->name,
5032 lkowner_utoa(&frame->root->lk_owner));
5033 }
5034
5035 call_count = afr_frame_return(frame);
5036 if (call_count == 0) {
5037 AFR_STACK_UNWIND(lk, frame, local->op_ret, local->op_errno, NULL,
5038 local->xdata_rsp);
5039 }
5040
5041 return 0;
5042 }
5043
5044 int32_t
5045 afr_lk_unlock(call_frame_t *frame, xlator_t *this)
5046 {
5047 afr_local_t *local = NULL;
5048 afr_private_t *priv = NULL;
5049 int i = 0;
5050 int call_count = 0;
5051
5052 local = frame->local;
5053 priv = this->private;
5054
5055 call_count = afr_locked_nodes_count(local->cont.lk.locked_nodes,
5056 priv->child_count);
5057
5058 if (call_count == 0) {
5059 AFR_STACK_UNWIND(lk, frame, local->op_ret, local->op_errno, NULL,
5060 local->xdata_rsp);
5061 return 0;
5062 }
5063
5064 local->call_count = call_count;
5065
5066 local->cont.lk.user_flock.l_type = F_UNLCK;
5067
5068 for (i = 0; i < priv->child_count; i++) {
5069 if (local->cont.lk.locked_nodes[i]) {
5070 STACK_WIND_COOKIE(frame, afr_lk_unlock_cbk, (void *)(long)i,
5071 priv->children[i], priv->children[i]->fops->lk,
5072 local->fd, F_SETLK, &local->cont.lk.user_flock,
5073 NULL);
5074
5075 if (!--call_count)
5076 break;
5077 }
5078 }
5079
5080 return 0;
5081 }
5082
5083 int32_t
5084 afr_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
5085 int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
5086 {
5087 afr_local_t *local = NULL;
5088 afr_private_t *priv = NULL;
5089 int child_index = -1;
5090
5091 local = frame->local;
5092 priv = this->private;
5093
5094 child_index = (long)cookie;
5095
5096 afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
5097
5098 /* We have two options here to handle EINTR
5099 1) Either Ignore EINTR and wind a call on other subvolume
5100 2) or Unlock the already granted lock in the previous children and return
5101 the error.
5102
5103 For EINTR we can't chose first option because EINTR is not a regular
5104 error. The user expecation is whole lock should be clear after receive an
5105 interrupt. However there is an issue if we do unlock already granted
5106 locks. Locks xlator implements posix locks as they are defined by the
5107 posix standard. This means that granted locks are "combined" with other
5108 existing granted locks. Unfortunately, in some cases this makes it
5109 impossible to revoke an already granted lock without unwanted side
5110 effects. For example consider a process that takes two read locks on the
5111 same file, but their ranges overlap a bit. Once both of them are granted
5112 in one brick, revoking any of them because of an interrupt will cause the
5113 overlapped range to be unlocked, even if one of the locks is still
5114 granted (from the point of view of the user, one of the locks has never
5115 been granted because it returned EINTR, so the user still expects that
5116 the other lock still covers the overlapping region). For the time being
5117 we have decided consider EINTR as same EAGAIN.
5118 */
5119
5120 if (op_ret < 0 && ((op_errno == EAGAIN) || (op_errno == EINTR))) {
5121 local->op_ret = -1;
5122 local->op_errno = op_errno;
5123
5124 afr_lk_unlock(frame, this);
5125 return 0;
5126 }
5127
5128 if (op_ret == 0) {
5129 local->op_ret = 0;
5130 local->op_errno = 0;
5131 local->cont.lk.locked_nodes[child_index] = 1;
5132 gf_flock_copy(&local->cont.lk.ret_flock, lock);
5133 }
5134
5135 child_index++;
5136
5137 if (child_index < priv->child_count) {
5138 STACK_WIND_COOKIE(frame, afr_lk_cbk, (void *)(long)child_index,
5139 priv->children[child_index],
5140 priv->children[child_index]->fops->lk, local->fd,
5141 local->cont.lk.cmd, &local->cont.lk.user_flock,
5142 local->xdata_req);
5143 } else if (priv->quorum_count &&
5144 !afr_has_quorum(local->cont.lk.locked_nodes, priv, NULL)) {
5145 local->op_ret = -1;
5146 local->op_errno = afr_final_errno(local, priv);
5147
5148 afr_lk_unlock(frame, this);
5149 } else {
5150 if (local->op_ret < 0)
5151 local->op_errno = afr_final_errno(local, priv);
5152
5153 AFR_STACK_UNWIND(lk, frame, local->op_ret, local->op_errno,
5154 &local->cont.lk.ret_flock, local->xdata_rsp);
5155 }
5156
5157 return 0;
5158 }
5159
5160 int
5161 afr_lk_transaction_cbk(int ret, call_frame_t *frame, void *opaque)
5162 {
5163 return 0;
5164 }
5165
5166 int
5167 afr_lk_txn_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
5168 int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
5169 dict_t *xdata)
5170 {
5171 afr_local_t *local = NULL;
5172 int child_index = -1;
5173
5174 local = frame->local;
5175 child_index = (long)cookie;
5176 afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
5177 if (op_ret == 0) {
5178 local->op_ret = 0;
5179 local->op_errno = 0;
5180 local->cont.lk.locked_nodes[child_index] = 1;
5181 gf_flock_copy(&local->cont.lk.ret_flock, lock);
5182 }
5183 syncbarrier_wake(&local->barrier);
5184 return 0;
5185 }
5186
5187 int
5188 afr_lk_txn_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
5189 int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
5190 dict_t *xdata)
5191 {
5192 afr_local_t *local = frame->local;
5193 afr_private_t *priv = this->private;
5194 int child_index = (long)cookie;
5195
5196 if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
5197 gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_UNLOCK_FAIL,
5198 "gfid=%s: unlock failed on subvolume %s "
5199 "with lock owner %s",
5200 uuid_utoa(local->fd->inode->gfid),
5201 priv->children[child_index]->name,
5202 lkowner_utoa(&frame->root->lk_owner));
5203 }
5204 return 0;
5205 }
5206 int
5207 afr_lk_transaction(void *opaque)
5208 {
5209 call_frame_t *frame = NULL;
5210 xlator_t *this = NULL;
5211 afr_private_t *priv = NULL;
5212 afr_local_t *local = NULL;
5213 char *wind_on = NULL;
5214 int op_errno = 0;
5215 int i = 0;
5216 int ret = 0;
5217
5218 frame = (call_frame_t *)opaque;
5219 local = frame->local;
5220 this = frame->this;
5221 priv = this->private;
5222 wind_on = alloca0(priv->child_count);
5223
5224 if (priv->arbiter_count || priv->child_count != 3) {
5225 op_errno = ENOTSUP;
5226 gf_msg(frame->this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
5227 "%s: Lock healing supported only for replica 3 volumes.",
5228 uuid_utoa(local->fd->inode->gfid));
5229 goto err;
5230 }
5231
5232 op_errno = -afr_dom_lock_acquire(frame); // Released during
5233 // AFR_STACK_UNWIND
5234 if (op_errno != 0) {
5235 goto err;
5236 }
5237 if (priv->quorum_count &&
5238 !afr_has_quorum(local->cont.lk.dom_locked_nodes, priv, NULL)) {
5239 op_errno = afr_final_errno(local, priv);
5240 goto err;
5241 }
5242
5243 for (i = 0; i < priv->child_count; i++) {
5244 if (priv->child_up[i] && local->cont.lk.dom_locked_nodes[i])
5245 wind_on[i] = 1;
5246 }
5247 AFR_ONLIST(wind_on, frame, afr_lk_txn_wind_cbk, lk, local->fd,
5248 local->cont.lk.cmd, &local->cont.lk.user_flock,
5249 local->xdata_req);
5250
5251 if (priv->quorum_count &&
5252 !afr_has_quorum(local->cont.lk.locked_nodes, priv, NULL)) {
5253 local->op_ret = -1;
5254 local->op_errno = afr_final_errno(local, priv);
5255 goto unlock;
5256 } else {
5257 if (local->cont.lk.user_flock.l_type == F_UNLCK)
5258 ret = afr_remove_lock_from_saved_locks(local, this);
5259 else
5260 ret = afr_add_lock_to_saved_locks(frame, this);
5261 if (ret) {
5262 local->op_ret = -1;
5263 local->op_errno = -ret;
5264 goto unlock;
5265 }
5266 AFR_STACK_UNWIND(lk, frame, local->op_ret, local->op_errno,
5267 &local->cont.lk.ret_flock, local->xdata_rsp);
5268 }
5269
5270 return 0;
5271
5272 unlock:
5273 local->cont.lk.user_flock.l_type = F_UNLCK;
5274 AFR_ONLIST(local->cont.lk.locked_nodes, frame, afr_lk_txn_unlock_cbk, lk,
5275 local->fd, F_SETLK, &local->cont.lk.user_flock, NULL);
5276 err:
5277 AFR_STACK_UNWIND(lk, frame, -1, op_errno, NULL, NULL);
5278 return -1;
5279 }
5280
5281 int
5282 afr_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
5283 struct gf_flock *flock, dict_t *xdata)
5284 {
5285 afr_private_t *priv = NULL;
5286 afr_local_t *local = NULL;
5287 int ret = 0;
5288 int i = 0;
5289 int32_t op_errno = ENOMEM;
5290
5291 priv = this->private;
5292
5293 local = AFR_FRAME_INIT(frame, op_errno);
5294 if (!local)
5295 goto out;
5296
5297 local->op = GF_FOP_LK;
5298 if (!afr_lk_is_unlock(cmd, flock)) {
5299 AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
5300 if (!afr_is_consistent_io_possible(local, priv, &op_errno))
5301 goto out;
5302 }
5303
5304 local->cont.lk.locked_nodes = GF_CALLOC(
5305 priv->child_count, sizeof(*local->cont.lk.locked_nodes),
5306 gf_afr_mt_char);
5307
5308 if (!local->cont.lk.locked_nodes) {
5309 op_errno = ENOMEM;
5310 goto out;
5311 }
5312
5313 local->fd = fd_ref(fd);
5314 local->cont.lk.cmd = cmd;
5315 gf_flock_copy(&local->cont.lk.user_flock, flock);
5316 gf_flock_copy(&local->cont.lk.ret_flock, flock);
5317 if (xdata) {
5318 local->xdata_req = dict_ref(xdata);
5319 if (afr_is_lock_mode_mandatory(xdata)) {
5320 ret = synctask_new(this->ctx->env, afr_lk_transaction,
5321 afr_lk_transaction_cbk, frame, frame);
5322 if (ret) {
5323 op_errno = ENOMEM;
5324 goto out;
5325 }
5326 return 0;
5327 }
5328 }
5329
5330 STACK_WIND_COOKIE(frame, afr_lk_cbk, (void *)(long)0, priv->children[i],
5331 priv->children[i]->fops->lk, fd, cmd, flock,
5332 local->xdata_req);
5333
5334 return 0;
5335 out:
5336 AFR_STACK_UNWIND(lk, frame, -1, op_errno, NULL, NULL);
5337
5338 return 0;
5339 }
5340
5341 static int32_t
5342 afr_lease_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
5343 int32_t op_ret, int32_t op_errno, struct gf_lease *lease,
5344 dict_t *xdata)
5345 {
5346 afr_local_t *local = NULL;
5347 int call_count = -1;
5348
5349 local = frame->local;
5350 call_count = afr_frame_return(frame);
5351
5352 if (call_count == 0)
5353 AFR_STACK_UNWIND(lease, frame, local->op_ret, local->op_errno, lease,
5354 xdata);
5355
5356 return 0;
5357 }
5358
5359 int32_t
5360 afr_lease_unlock(call_frame_t *frame, xlator_t *this)
5361 {
5362 afr_local_t *local = NULL;
5363 afr_private_t *priv = NULL;
5364 int i = 0;
5365 int call_count = 0;
5366
5367 local = frame->local;
5368 priv = this->private;
5369
5370 call_count = afr_locked_nodes_count(local->cont.lease.locked_nodes,
5371 priv->child_count);
5372
5373 if (call_count == 0) {
5374 AFR_STACK_UNWIND(lease, frame, local->op_ret, local->op_errno,
5375 &local->cont.lease.ret_lease, NULL);
5376 return 0;
5377 }
5378
5379 local->call_count = call_count;
5380
5381 local->cont.lease.user_lease.cmd = GF_UNLK_LEASE;
5382
5383 for (i = 0; i < priv->child_count; i++) {
5384 if (local->cont.lease.locked_nodes[i]) {
5385 STACK_WIND(frame, afr_lease_unlock_cbk, priv->children[i],
5386 priv->children[i]->fops->lease, &local->loc,
5387 &local->cont.lease.user_lease, NULL);
5388
5389 if (!--call_count)
5390 break;
5391 }
5392 }
5393
5394 return 0;
5395 }
5396
5397 int32_t
5398 afr_lease_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
5399 int32_t op_errno, struct gf_lease *lease, dict_t *xdata)
5400 {
5401 afr_local_t *local = NULL;
5402 afr_private_t *priv = NULL;
5403 int child_index = -1;
5404
5405 local = frame->local;
5406 priv = this->private;
5407
5408 child_index = (long)cookie;
5409
5410 afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
5411 if (op_ret < 0 && op_errno == EAGAIN) {
5412 local->op_ret = -1;
5413 local->op_errno = EAGAIN;
5414
5415 afr_lease_unlock(frame, this);
5416 return 0;
5417 }
5418
5419 if (op_ret == 0) {
5420 local->op_ret = 0;
5421 local->op_errno = 0;
5422 local->cont.lease.locked_nodes[child_index] = 1;
5423 local->cont.lease.ret_lease = *lease;
5424 }
5425
5426 child_index++;
5427 if (child_index < priv->child_count) {
5428 STACK_WIND_COOKIE(frame, afr_lease_cbk, (void *)(long)child_index,
5429 priv->children[child_index],
5430 priv->children[child_index]->fops->lease, &local->loc,
5431 &local->cont.lease.user_lease, xdata);
5432 } else if (priv->quorum_count &&
5433 !afr_has_quorum(local->cont.lease.locked_nodes, priv, NULL)) {
5434 local->op_ret = -1;
5435 local->op_errno = afr_final_errno(local, priv);
5436
5437 afr_lease_unlock(frame, this);
5438 } else {
5439 if (local->op_ret < 0)
5440 local->op_errno = afr_final_errno(local, priv);
5441 AFR_STACK_UNWIND(lease, frame, local->op_ret, local->op_errno,
5442 &local->cont.lease.ret_lease, NULL);
5443 }
5444
5445 return 0;
5446 }
5447
5448 int
5449 afr_lease(call_frame_t *frame, xlator_t *this, loc_t *loc,
5450 struct gf_lease *lease, dict_t *xdata)
5451 {
5452 afr_private_t *priv = NULL;
5453 afr_local_t *local = NULL;
5454 int32_t op_errno = ENOMEM;
5455
5456 priv = this->private;
5457
5458 local = AFR_FRAME_INIT(frame, op_errno);
5459 if (!local)
5460 goto out;
5461
5462 local->op = GF_FOP_LEASE;
5463 local->cont.lease.locked_nodes = GF_CALLOC(
5464 priv->child_count, sizeof(*local->cont.lease.locked_nodes),
5465 gf_afr_mt_char);
5466
5467 if (!local->cont.lease.locked_nodes) {
5468 op_errno = ENOMEM;
5469 goto out;
5470 }
5471
5472 loc_copy(&local->loc, loc);
5473 local->cont.lease.user_lease = *lease;
5474 local->cont.lease.ret_lease = *lease;
5475
5476 STACK_WIND_COOKIE(frame, afr_lease_cbk, (void *)(long)0, priv->children[0],
5477 priv->children[0]->fops->lease, loc, lease, xdata);
5478
5479 return 0;
5480 out:
5481 AFR_STACK_UNWIND(lease, frame, -1, op_errno, NULL, NULL);
5482
5483 return 0;
5484 }
5485
5486 int
5487 afr_ipc_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
5488 int32_t op_errno, dict_t *xdata)
5489 {
5490 afr_local_t *local = NULL;
5491 int child_index = (long)cookie;
5492 int call_count = 0;
5493 gf_boolean_t failed = _gf_false;
5494 gf_boolean_t succeeded = _gf_false;
5495 int i = 0;
5496 afr_private_t *priv = NULL;
5497
5498 local = frame->local;
5499 priv = this->private;
5500
5501 local->replies[child_index].valid = 1;
5502 local->replies[child_index].op_ret = op_ret;
5503 local->replies[child_index].op_errno = op_errno;
5504 if (xdata)
5505 local->replies[child_index].xdata = dict_ref(xdata);
5506
5507 call_count = afr_frame_return(frame);
5508 if (call_count)
5509 goto out;
5510 /* If any of the subvolumes failed with other than ENOTCONN
5511 * return error else return success unless all the subvolumes
5512 * failed.
5513 * TODO: In case of failure, we need to unregister the xattrs
5514 * from the other subvolumes where it succeeded (once upcall
5515 * fixes the Bz-1371622)*/
5516 for (i = 0; i < priv->child_count; i++) {
5517 if (!local->replies[i].valid)
5518 continue;
5519 if (local->replies[i].op_ret < 0 &&
5520 local->replies[i].op_errno != ENOTCONN) {
5521 local->op_ret = local->replies[i].op_ret;
5522 local->op_errno = local->replies[i].op_errno;
5523 if (local->xdata_rsp)
5524 dict_unref(local->xdata_rsp);
5525 local->xdata_rsp = NULL;
5526 if (local->replies[i].xdata) {
5527 local->xdata_rsp = dict_ref(local->replies[i].xdata);
5528 }
5529 failed = _gf_true;
5530 break;
5531 }
5532 if (local->replies[i].op_ret == 0) {
5533 succeeded = _gf_true;
5534 local->op_ret = 0;
5535 local->op_errno = 0;
5536 if (!local->xdata_rsp && local->replies[i].xdata) {
5537 local->xdata_rsp = dict_ref(local->replies[i].xdata);
5538 }
5539 }
5540 }
5541
5542 if (!succeeded && !failed) {
5543 local->op_ret = -1;
5544 local->op_errno = ENOTCONN;
5545 }
5546
5547 AFR_STACK_UNWIND(ipc, frame, local->op_ret, local->op_errno,
5548 local->xdata_rsp);
5549
5550 out:
5551 return 0;
5552 }
5553
5554 int
5555 afr_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
5556 {
5557 afr_local_t *local = NULL;
5558 int32_t op_errno = -1;
5559 afr_private_t *priv = NULL;
5560 int i = 0;
5561 int call_cnt = -1;
5562
5563 VALIDATE_OR_GOTO(frame, err);
5564 VALIDATE_OR_GOTO(this, err);
5565
5566 if (op != GF_IPC_TARGET_UPCALL)
5567 goto wind_default;
5568
5569 VALIDATE_OR_GOTO(this->private, err);
5570 priv = this->private;
5571
5572 local = AFR_FRAME_INIT(frame, op_errno);
5573 if (!local)
5574 goto err;
5575
5576 call_cnt = local->call_count;
5577
5578 if (xdata) {
5579 for (i = 0; i < priv->child_count; i++) {
5580 if (dict_set_int8(xdata, priv->pending_key[i], 0) < 0)
5581 goto err;
5582 }
5583 }
5584
5585 for (i = 0; i < priv->child_count; i++) {
5586 if (!local->child_up[i])
5587 continue;
5588
5589 STACK_WIND_COOKIE(frame, afr_ipc_cbk, (void *)(long)i,
5590 priv->children[i], priv->children[i]->fops->ipc, op,
5591 xdata);
5592 if (!--call_cnt)
5593 break;
5594 }
5595 return 0;
5596
5597 err:
5598 if (op_errno == -1)
5599 op_errno = errno;
5600 AFR_STACK_UNWIND(ipc, frame, -1, op_errno, NULL);
5601
5602 return 0;
5603
5604 wind_default:
5605 STACK_WIND(frame, default_ipc_cbk, FIRST_CHILD(this),
5606 FIRST_CHILD(this)->fops->ipc, op, xdata);
5607 return 0;
5608 }
5609
5610 int
5611 afr_forget(xlator_t *this, inode_t *inode)
5612 {
5613 uint64_t ctx_int = 0;
5614 afr_inode_ctx_t *ctx = NULL;
5615
5616 afr_spb_choice_timeout_cancel(this, inode);
5617 inode_ctx_del(inode, this, &ctx_int);
5618 if (!ctx_int)
5619 return 0;
5620
5621 ctx = (afr_inode_ctx_t *)(uintptr_t)ctx_int;
5622 afr_inode_ctx_destroy(ctx);
5623 return 0;
5624 }
5625
5626 int
5627 afr_priv_dump(xlator_t *this)
5628 {
5629 afr_private_t *priv = NULL;
5630 char key_prefix[GF_DUMP_MAX_BUF_LEN];
5631 char key[GF_DUMP_MAX_BUF_LEN];
5632 int i = 0;
5633
5634 GF_ASSERT(this);
5635 priv = this->private;
5636
5637 GF_ASSERT(priv);
5638 snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type, this->name);
5639 gf_proc_dump_add_section("%s", key_prefix);
5640 gf_proc_dump_write("child_count", "%u", priv->child_count);
5641 for (i = 0; i < priv->child_count; i++) {
5642 sprintf(key, "child_up[%d]", i);
5643 gf_proc_dump_write(key, "%d", priv->child_up[i]);
5644 sprintf(key, "pending_key[%d]", i);
5645 gf_proc_dump_write(key, "%s", priv->pending_key[i]);
5646 sprintf(key, "pending_reads[%d]", i);
5647 gf_proc_dump_write(key, "%" PRId64,
5648 GF_ATOMIC_GET(priv->pending_reads[i]));
5649 sprintf(key, "child_latency[%d]", i);
5650 gf_proc_dump_write(key, "%" PRId64, priv->child_latency[i]);
5651 sprintf(key, "halo_child_up[%d]", i);
5652 gf_proc_dump_write(key, "%d", priv->halo_child_up[i]);
5653 }
5654 gf_proc_dump_write("data_self_heal", "%d", priv->data_self_heal);
5655 gf_proc_dump_write("metadata_self_heal", "%d", priv->metadata_self_heal);
5656 gf_proc_dump_write("entry_self_heal", "%d", priv->entry_self_heal);
5657 gf_proc_dump_write("read_child", "%d", priv->read_child);
5658 gf_proc_dump_write("wait_count", "%u", priv->wait_count);
5659 gf_proc_dump_write("heal-wait-queue-length", "%d", priv->heal_wait_qlen);
5660 gf_proc_dump_write("heal-waiters", "%d", priv->heal_waiters);
5661 gf_proc_dump_write("background-self-heal-count", "%d",
5662 priv->background_self_heal_count);
5663 gf_proc_dump_write("healers", "%d", priv->healers);
5664 gf_proc_dump_write("read-hash-mode", "%d", priv->hash_mode);
5665 gf_proc_dump_write("use-anonymous-inode", "%d", priv->use_anon_inode);
5666 if (priv->quorum_count == AFR_QUORUM_AUTO) {
5667 gf_proc_dump_write("quorum-type", "auto");
5668 } else if (priv->quorum_count == 0) {
5669 gf_proc_dump_write("quorum-type", "none");
5670 } else {
5671 gf_proc_dump_write("quorum-type", "fixed");
5672 gf_proc_dump_write("quorum-count", "%d", priv->quorum_count);
5673 }
5674 gf_proc_dump_write("up", "%u", afr_has_quorum(priv->child_up, priv, NULL));
5675 if (priv->thin_arbiter_count) {
5676 gf_proc_dump_write("ta_child_up", "%d", priv->ta_child_up);
5677 gf_proc_dump_write("ta_bad_child_index", "%d",
5678 priv->ta_bad_child_index);
5679 gf_proc_dump_write("ta_notify_dom_lock_offset", "%" PRId64,
5680 priv->ta_notify_dom_lock_offset);
5681 }
5682
5683 return 0;
5684 }
5685
5686 /**
5687 * find_child_index - find the child's index in the array of subvolumes
5688 * @this: AFR
5689 * @child: child
5690 */
5691
5692 static int
5693 afr_find_child_index(xlator_t *this, xlator_t *child)
5694 {
5695 afr_private_t *priv = NULL;
5696 int child_count = -1;
5697 int i = -1;
5698
5699 priv = this->private;
5700 child_count = priv->child_count;
5701 if (priv->thin_arbiter_count) {
5702 child_count++;
5703 }
5704
5705 for (i = 0; i < child_count; i++) {
5706 if ((xlator_t *)child == priv->children[i])
5707 break;
5708 }
5709
5710 return i;
5711 }
5712
5713 int
5714 __afr_get_up_children_count(afr_private_t *priv)
5715 {
5716 int up_children = 0;
5717 int i = 0;
5718
5719 for (i = 0; i < priv->child_count; i++)
5720 if (priv->child_up[i] == 1)
5721 up_children++;
5722
5723 return up_children;
5724 }
5725
5726 static int
5727 __get_heard_from_all_status(afr_private_t *priv)
5728 {
5729 int i;
5730
5731 for (i = 0; i < priv->child_count; i++) {
5732 if (!priv->last_event[i]) {
5733 return 0;
5734 }
5735 }
5736 if (priv->thin_arbiter_count && !priv->ta_child_up) {
5737 return 0;
5738 }
5739 return 1;
5740 }
5741
5742 static glusterfs_event_t
5743 __afr_transform_event_from_state(afr_private_t *priv)
5744 {
5745 int i = 0;
5746 int up_children = 0;
5747
5748 if (__get_heard_from_all_status(priv))
5749 /* have_heard_from_all. Let afr_notify() do the propagation. */
5750 return GF_EVENT_MAXVAL;
5751
5752 up_children = __afr_get_up_children_count(priv);
5753 /* Treat the children with pending notification, as having sent a
5754 * GF_EVENT_CHILD_DOWN. i.e. set the event as GF_EVENT_SOME_DESCENDENT_DOWN,
5755 * as done in afr_notify() */
5756 for (i = 0; i < priv->child_count; i++) {
5757 if (priv->last_event[i])
5758 continue;
5759 priv->last_event[i] = GF_EVENT_SOME_DESCENDENT_DOWN;
5760 priv->child_up[i] = 0;
5761 }
5762
5763 if (up_children)
5764 /* We received at least one child up */
5765 return GF_EVENT_CHILD_UP;
5766 else
5767 return GF_EVENT_CHILD_DOWN;
5768
5769 return GF_EVENT_MAXVAL;
5770 }
5771
5772 static void
5773 afr_notify_cbk(void *data)
5774 {
5775 xlator_t *this = data;
5776 afr_private_t *priv = this->private;
5777 glusterfs_event_t event = GF_EVENT_MAXVAL;
5778 gf_boolean_t propagate = _gf_false;
5779
5780 LOCK(&priv->lock);
5781 {
5782 if (!priv->timer) {
5783 /*
5784 * Either child_up/child_down is already sent to parent.
5785 * This is a spurious wake up.
5786 */
5787 goto unlock;
5788 }
5789 priv->timer = NULL;
5790 event = __afr_transform_event_from_state(priv);
5791 if (event != GF_EVENT_MAXVAL)
5792 propagate = _gf_true;
5793 }
5794 unlock:
5795 UNLOCK(&priv->lock);
5796 if (propagate)
5797 default_notify(this, event, NULL);
5798 }
5799
5800 static void
5801 __afr_launch_notify_timer(xlator_t *this, afr_private_t *priv)
5802 {
5803 struct timespec delay = {
5804 0,
5805 };
5806
5807 gf_msg_debug(this->name, 0, "Initiating child-down timer");
5808 delay.tv_sec = 10;
5809 delay.tv_nsec = 0;
5810 priv->timer = gf_timer_call_after(this->ctx, delay, afr_notify_cbk, this);
5811 if (priv->timer == NULL) {
5812 gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_TIMER_CREATE_FAIL,
5813 "Cannot create timer for delayed initialization");
5814 }
5815 }
5816
5817 static int
5818 find_best_down_child(xlator_t *this)
5819 {
5820 afr_private_t *priv = NULL;
5821 int i = -1;
5822 int32_t best_child = -1;
5823 int64_t best_latency = INT64_MAX;
5824
5825 priv = this->private;
5826
5827 for (i = 0; i < priv->child_count; i++) {
5828 if (!priv->child_up[i] && priv->child_latency[i] >= 0 &&
5829 priv->child_latency[i] < best_latency) {
5830 best_child = i;
5831 best_latency = priv->child_latency[i];
5832 }
5833 }
5834 if (best_child >= 0) {
5835 gf_msg_debug(this->name, 0,
5836 "Found best down child (%d) @ %" PRId64 " ms latency",
5837 best_child, best_latency);
5838 }
5839 return best_child;
5840 }
5841
5842 static int
5843 find_worst_up_child(xlator_t *this)
5844 {
5845 afr_private_t *priv = NULL;
5846 int i = -1;
5847 int32_t worst_child = -1;
5848 int64_t worst_latency = INT64_MIN;
5849
5850 priv = this->private;
5851
5852 for (i = 0; i < priv->child_count; i++) {
5853 if (priv->child_up[i] && priv->child_latency[i] >= 0 &&
5854 priv->child_latency[i] > worst_latency) {
5855 worst_child = i;
5856 worst_latency = priv->child_latency[i];
5857 }
5858 }
5859 if (worst_child >= 0) {
5860 gf_msg_debug(this->name, 0,
5861 "Found worst up child (%d) @ %" PRId64 " ms latency",
5862 worst_child, worst_latency);
5863 }
5864 return worst_child;
5865 }
5866
5867 static void
5868 __afr_handle_ping_event(xlator_t *this, xlator_t *child_xlator, const int idx,
5869 int64_t halo_max_latency_msec, int32_t *event,
5870 int64_t child_latency_msec)
5871 {
5872 afr_private_t *priv = NULL;
5873 int up_children = 0;
5874
5875 priv = this->private;
5876
5877 priv->child_latency[idx] = child_latency_msec;
5878 gf_msg_debug(child_xlator->name, 0, "Client ping @ %" PRId64 " ms",
5879 child_latency_msec);
5880 if (priv->shd.iamshd)
5881 return;
5882
5883 up_children = __afr_get_up_children_count(priv);
5884
5885 if (child_latency_msec > halo_max_latency_msec &&
5886 priv->child_up[idx] == 1 && up_children > priv->halo_min_replicas) {
5887 if ((up_children - 1) < priv->halo_min_replicas) {
5888 gf_log(child_xlator->name, GF_LOG_INFO,
5889 "Overriding halo threshold, "
5890 "min replicas: %d",
5891 priv->halo_min_replicas);
5892 } else {
5893 gf_log(child_xlator->name, GF_LOG_INFO,
5894 "Child latency (%" PRId64
5895 " ms) "
5896 "exceeds halo threshold (%" PRId64
5897 "), "
5898 "marking child down.",
5899 child_latency_msec, halo_max_latency_msec);
5900 if (priv->halo_child_up[idx]) {
5901 *event = GF_EVENT_CHILD_DOWN;
5902 }
5903 }
5904 } else if (child_latency_msec < halo_max_latency_msec &&
5905 priv->child_up[idx] == 0) {
5906 if (up_children < priv->halo_max_replicas) {
5907 gf_log(child_xlator->name, GF_LOG_INFO,
5908 "Child latency (%" PRId64
5909 " ms) "
5910 "below halo threshold (%" PRId64
5911 "), "
5912 "marking child up.",
5913 child_latency_msec, halo_max_latency_msec);
5914 if (priv->halo_child_up[idx]) {
5915 *event = GF_EVENT_CHILD_UP;
5916 }
5917 } else {
5918 gf_log(child_xlator->name, GF_LOG_INFO,
5919 "Not marking child %d up, "
5920 "max replicas (%d) reached.",
5921 idx, priv->halo_max_replicas);
5922 }
5923 }
5924 }
5925
5926 static int64_t
5927 afr_get_halo_latency(xlator_t *this)
5928 {
5929 afr_private_t *priv = NULL;
5930 int64_t halo_max_latency_msec = 0;
5931
5932 priv = this->private;
5933
5934 if (priv->shd.iamshd) {
5935 halo_max_latency_msec = priv->shd.halo_max_latency_msec;
5936 } else if (priv->nfsd.iamnfsd) {
5937 halo_max_latency_msec = priv->nfsd.halo_max_latency_msec;
5938 } else {
5939 halo_max_latency_msec = priv->halo_max_latency_msec;
5940 }
5941 gf_msg_debug(this->name, 0, "Using halo latency %" PRId64,
5942 halo_max_latency_msec);
5943 return halo_max_latency_msec;
5944 }
5945
5946 static void
5947 __afr_handle_child_up_event(xlator_t *this, xlator_t *child_xlator,
5948 const int idx, int64_t child_latency_msec,
5949 int32_t *event, int32_t *call_psh,
5950 int32_t *up_child)
5951 {
5952 afr_private_t *priv = NULL;
5953 int up_children = 0;
5954 int worst_up_child = -1;
5955 int64_t halo_max_latency_msec = afr_get_halo_latency(this);
5956
5957 priv = this->private;
5958
5959 /*
5960 * This only really counts if the child was never up
5961 * (value = -1) or had been down (value = 0). See
5962 * comment at GF_EVENT_CHILD_DOWN for a more detailed
5963 * explanation.
5964 */
5965 if (priv->child_up[idx] != 1) {
5966 priv->event_generation++;
5967 }
5968 priv->child_up[idx] = 1;
5969
5970 *call_psh = 1;
5971 *up_child = idx;
5972 up_children = __afr_get_up_children_count(priv);
5973 /*
5974 * If this is an _actual_ CHILD_UP event, we
5975 * want to set the child_latency to MAX to indicate
5976 * the child needs ping data to be available before doing child-up
5977 */
5978 if (!priv->halo_enabled)
5979 goto out;
5980
5981 if (child_latency_msec < 0) {
5982 /*set to INT64_MAX-1 so that it is found for best_down_child*/
5983 priv->halo_child_up[idx] = 1;
5984 if (priv->child_latency[idx] < 0) {
5985 priv->child_latency[idx] = AFR_HALO_MAX_LATENCY;
5986 }
5987 }
5988
5989 /*
5990 * Handle the edge case where we exceed
5991 * halo_min_replicas and we've got a child which is
5992 * marked up as it was helping to satisfy the
5993 * halo_min_replicas even though it's latency exceeds
5994 * halo_max_latency_msec.
5995 */
5996 if (up_children > priv->halo_min_replicas) {
5997 worst_up_child = find_worst_up_child(this);
5998 if (worst_up_child >= 0 &&
5999 priv->child_latency[worst_up_child] > halo_max_latency_msec) {
6000 gf_msg_debug(this->name, 0,
6001 "Marking child %d down, "
6002 "doesn't meet halo threshold (%" PRId64
6003 "), and > "
6004 "halo_min_replicas (%d)",
6005 worst_up_child, halo_max_latency_msec,
6006 priv->halo_min_replicas);
6007 priv->child_up[worst_up_child] = 0;
6008 up_children--;
6009 }
6010 }
6011
6012 if (up_children > priv->halo_max_replicas && !priv->shd.iamshd) {
6013 worst_up_child = find_worst_up_child(this);
6014 if (worst_up_child < 0) {
6015 worst_up_child = idx;
6016 }
6017 priv->child_up[worst_up_child] = 0;
6018 up_children--;
6019 gf_msg_debug(this->name, 0,
6020 "Marking child %d down, "
6021 "up_children (%d) > halo_max_replicas (%d)",
6022 worst_up_child, up_children, priv->halo_max_replicas);
6023 }
6024 out:
6025 if (up_children == 1) {
6026 gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SUBVOL_UP,
6027 "Subvolume '%s' came back up; "
6028 "going online.",
6029 child_xlator->name);
6030 gf_event(EVENT_AFR_SUBVOL_UP, "client-pid=%d; subvol=%s",
6031 this->ctx->cmd_args.client_pid, this->name);
6032 } else {
6033 *event = GF_EVENT_SOME_DESCENDENT_UP;
6034 }
6035
6036 priv->last_event[idx] = *event;
6037 }
6038
6039 void
6040 __afr_handle_child_down_event(xlator_t *this, xlator_t *child_xlator, int idx,
6041 int64_t child_latency_msec, int32_t *event,
6042 int32_t *call_psh, int32_t *up_child)
6043 {
6044 afr_private_t *priv = NULL;
6045 int i = 0;
6046 int up_children = 0;
6047 int down_children = 0;
6048 int best_down_child = -1;
6049
6050 priv = this->private;
6051
6052 /*
6053 * If a brick is down when we start, we'll get a
6054 * CHILD_DOWN to indicate its initial state. There
6055 * was never a CHILD_UP in this case, so if we
6056 * increment "down_count" the difference between than
6057 * and "up_count" will no longer be the number of
6058 * children that are currently up. This has serious
6059 * implications e.g. for quorum enforcement, so we
6060 * don't increment these values unless the event
6061 * represents an actual state transition between "up"
6062 * (value = 1) and anything else.
6063 */
6064 if (priv->child_up[idx] == 1) {
6065 priv->event_generation++;
6066 }
6067
6068 /*
6069 * If this is an _actual_ CHILD_DOWN event, we
6070 * want to set the child_latency to < 0 to indicate
6071 * the child is really disconnected.
6072 */
6073 if (child_latency_msec < 0) {
6074 priv->child_latency[idx] = child_latency_msec;
6075 priv->halo_child_up[idx] = 0;
6076 }
6077 priv->child_up[idx] = 0;
6078
6079 up_children = __afr_get_up_children_count(priv);
6080 /*
6081 * Handle the edge case where we need to find the
6082 * next best child (to mark up) as marking this child
6083 * down would cause us to fall below halo_min_replicas.
6084 * We will also force the SHD to heal this child _now_
6085 * as we want it to be up to date if we are going to
6086 * begin using it synchronously.
6087 */
6088 if (priv->halo_enabled && up_children < priv->halo_min_replicas) {
6089 best_down_child = find_best_down_child(this);
6090 if (best_down_child >= 0) {
6091 gf_msg_debug(this->name, 0,
6092 "Swapping out child %d for "
6093 "child %d to satisfy halo_min_replicas (%d).",
6094 idx, best_down_child, priv->halo_min_replicas);
6095 priv->child_up[best_down_child] = 1;
6096 *call_psh = 1;
6097 *up_child = best_down_child;
6098 }
6099 }
6100 for (i = 0; i < priv->child_count; i++)
6101 if (priv->child_up[i] == 0)
6102 down_children++;
6103 if (down_children == priv->child_count) {
6104 gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SUBVOLS_DOWN,
6105 "All subvolumes are down. Going "
6106 "offline until at least one of them "
6107 "comes back up.");
6108 gf_event(EVENT_AFR_SUBVOLS_DOWN, "client-pid=%d; subvol=%s",
6109 this->ctx->cmd_args.client_pid, this->name);
6110 } else {
6111 *event = GF_EVENT_SOME_DESCENDENT_DOWN;
6112 }
6113 priv->last_event[idx] = *event;
6114 }
6115
6116 void
6117 afr_ta_lock_release_synctask(xlator_t *this)
6118 {
6119 call_frame_t *ta_frame = NULL;
6120 int ret = 0;
6121
6122 ta_frame = afr_ta_frame_create(this);
6123 if (!ta_frame) {
6124 gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
6125 "Failed to create ta_frame");
6126 return;
6127 }
6128
6129 ret = synctask_new(this->ctx->env, afr_release_notify_lock_for_ta,
6130 afr_ta_lock_release_done, ta_frame, this);
6131 if (ret) {
6132 STACK_DESTROY(ta_frame->root);
6133 gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
6134 "Failed to release "
6135 "AFR_TA_DOM_NOTIFY lock.");
6136 }
6137 }
6138
6139 static void
6140 afr_handle_inodelk_contention(xlator_t *this, struct gf_upcall *upcall)
6141 {
6142 struct gf_upcall_inodelk_contention *lc = NULL;
6143 unsigned int inmem_count = 0;
6144 unsigned int onwire_count = 0;
6145 afr_private_t *priv = this->private;
6146
6147 lc = upcall->data;
6148
6149 if (strcmp(lc->domain, AFR_TA_DOM_NOTIFY) != 0)
6150 return;
6151
6152 if (priv->shd.iamshd) {
6153 /* shd should ignore AFR_TA_DOM_NOTIFY release requests. */
6154 return;
6155 }
6156 LOCK(&priv->lock);
6157 {
6158 if (priv->release_ta_notify_dom_lock == _gf_true) {
6159 /* Ignore multiple release requests from shds.*/
6160 UNLOCK(&priv->lock);
6161 return;
6162 }
6163 priv->release_ta_notify_dom_lock = _gf_true;
6164 inmem_count = priv->ta_in_mem_txn_count;
6165 onwire_count = priv->ta_on_wire_txn_count;
6166 }
6167 UNLOCK(&priv->lock);
6168 if (inmem_count || onwire_count)
6169 /* lock release will happen in txn code path after
6170 * in-memory or on-wire txns are over.*/
6171 return;
6172
6173 afr_ta_lock_release_synctask(this);
6174 }
6175
6176 static void
6177 afr_handle_upcall_event(xlator_t *this, struct gf_upcall *upcall, const int idx)
6178 {
6179 struct gf_upcall_cache_invalidation *up_ci = NULL;
6180 afr_private_t *priv = this->private;
6181 inode_t *inode = NULL;
6182 inode_table_t *itable = NULL;
6183 int i = 0;
6184
6185 switch (upcall->event_type) {
6186 case GF_UPCALL_INODELK_CONTENTION:
6187 afr_handle_inodelk_contention(this, upcall);
6188 break;
6189 case GF_UPCALL_CACHE_INVALIDATION:
6190 up_ci = (struct gf_upcall_cache_invalidation *)upcall->data;
6191
6192 if (AFR_IS_ARBITER_BRICK(priv, idx)) {
6193 /* Skip all update involving iatt from arbiter
6194 * node, since the size contains invalid info
6195 */
6196 up_ci->flags &= ~UP_ATTR_FLAGS;
6197 up_ci->flags &= ~UP_PARENT_DENTRY_FLAGS;
6198 }
6199 /* Since md-cache will be aggressively filtering
6200 * lookups, the stale read issue will be more
6201 * pronounced. Hence when a pending xattr is set notify
6202 * all the md-cache clients to invalidate the existing
6203 * stat cache and send the lookup next time */
6204 if (!up_ci->dict)
6205 break;
6206 for (i = 0; i < priv->child_count; i++) {
6207 if (!dict_get(up_ci->dict, priv->pending_key[i]))
6208 continue;
6209 up_ci->flags |= UP_INVAL_ATTR;
6210 itable = ((xlator_t *)this->graph->top)->itable;
6211 /*Internal processes may not have itable for
6212 *top xlator*/
6213 if (itable)
6214 inode = inode_find(itable, upcall->gfid);
6215 if (inode)
6216 afr_inode_need_refresh_set(inode, this);
6217 break;
6218 }
6219 break;
6220 default:
6221 break;
6222 }
6223 }
6224
6225 int32_t
6226 afr_notify(xlator_t *this, int32_t event, void *data, void *data2)
6227 {
6228 afr_private_t *priv = NULL;
6229 xlator_t *child_xlator = NULL;
6230 int i = -1;
6231 int propagate = 0;
6232 int had_heard_from_all = 0;
6233 int have_heard_from_all = 0;
6234 int idx = -1;
6235 int ret = -1;
6236 int call_psh = 0;
6237 int up_child = -1;
6238 dict_t *input = NULL;
6239 dict_t *output = NULL;
6240 gf_boolean_t had_quorum = _gf_false;
6241 gf_boolean_t has_quorum = _gf_false;
6242 int64_t halo_max_latency_msec = 0;
6243 int64_t child_latency_msec = -1;
6244
6245 child_xlator = (xlator_t *)data;
6246
6247 priv = this->private;
6248
6249 if (!priv)
6250 return 0;
6251
6252 /*
6253 * We need to reset this in case children come up in "staggered"
6254 * fashion, so that we discover a late-arriving local subvolume. Note
6255 * that we could end up issuing N lookups to the first subvolume, and
6256 * O(N^2) overall, but N is small for AFR so it shouldn't be an issue.
6257 */
6258 priv->did_discovery = _gf_false;
6259
6260 /* parent xlators don't need to know about every child_up, child_down
6261 * because of afr ha. If all subvolumes go down, child_down has
6262 * to be triggered. In that state when 1 subvolume comes up child_up
6263 * needs to be triggered. dht optimizes revalidate lookup by sending
6264 * it only to one of its subvolumes. When child up/down happens
6265 * for afr's subvolumes dht should be notified by child_modified. The
6266 * subsequent revalidate lookup happens on all the dht's subvolumes
6267 * which triggers afr self-heals if any.
6268 */
6269 idx = afr_find_child_index(this, child_xlator);
6270 if (idx < 0) {
6271 gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_CHILD_UP,
6272 "Received child_up from invalid subvolume");
6273 goto out;
6274 }
6275
6276 had_quorum = priv->quorum_count &&
6277 afr_has_quorum(priv->child_up, priv, NULL);
6278 if (event == GF_EVENT_CHILD_PING) {
6279 child_latency_msec = (int64_t)(uintptr_t)data2;
6280 if (priv->halo_enabled) {
6281 halo_max_latency_msec = afr_get_halo_latency(this);
6282
6283 /* Calculates the child latency and sets event
6284 */
6285 LOCK(&priv->lock);
6286 {
6287 __afr_handle_ping_event(this, child_xlator, idx,
6288 halo_max_latency_msec, &event,
6289 child_latency_msec);
6290 }
6291 UNLOCK(&priv->lock);
6292 } else {
6293 LOCK(&priv->lock);
6294 {
6295 priv->child_latency[idx] = child_latency_msec;
6296 }
6297 UNLOCK(&priv->lock);
6298 }
6299 }
6300
6301 if (event == GF_EVENT_CHILD_PING) {
6302 /* This is the only xlator that handles PING, no reason to
6303 * propagate.
6304 */
6305 goto out;
6306 }
6307
6308 if (event == GF_EVENT_TRANSLATOR_OP) {
6309 LOCK(&priv->lock);
6310 {
6311 had_heard_from_all = __get_heard_from_all_status(priv);
6312 }
6313 UNLOCK(&priv->lock);
6314
6315 if (!had_heard_from_all) {
6316 ret = -1;
6317 } else {
6318 input = data;
6319 output = data2;
6320 ret = afr_xl_op(this, input, output);
6321 }
6322 goto out;
6323 }
6324
6325 if (event == GF_EVENT_UPCALL) {
6326 afr_handle_upcall_event(this, data, idx);
6327 }
6328
6329 LOCK(&priv->lock);
6330 {
6331 had_heard_from_all = __get_heard_from_all_status(priv);
6332 switch (event) {
6333 case GF_EVENT_PARENT_UP:
6334 __afr_launch_notify_timer(this, priv);
6335 propagate = 1;
6336 break;
6337 case GF_EVENT_CHILD_UP:
6338 if (priv->thin_arbiter_count &&
6339 (idx == AFR_CHILD_THIN_ARBITER)) {
6340 priv->ta_child_up = 1;
6341 priv->ta_event_gen++;
6342 break;
6343 }
6344 __afr_handle_child_up_event(this, child_xlator, idx,
6345 child_latency_msec, &event,
6346 &call_psh, &up_child);
6347 __afr_lock_heal_synctask(this, priv, idx);
6348 break;
6349
6350 case GF_EVENT_CHILD_DOWN:
6351 if (priv->thin_arbiter_count &&
6352 (idx == AFR_CHILD_THIN_ARBITER)) {
6353 priv->ta_child_up = 0;
6354 priv->ta_event_gen++;
6355 afr_ta_locked_priv_invalidate(priv);
6356 break;
6357 }
6358 __afr_handle_child_down_event(this, child_xlator, idx,
6359 child_latency_msec, &event,
6360 &call_psh, &up_child);
6361 __afr_mark_pending_lk_heal(this, priv, idx);
6362 break;
6363
6364 case GF_EVENT_CHILD_CONNECTING:
6365 priv->last_event[idx] = event;
6366
6367 break;
6368
6369 case GF_EVENT_SOME_DESCENDENT_DOWN:
6370 priv->last_event[idx] = event;
6371 break;
6372 default:
6373 propagate = 1;
6374 break;
6375 }
6376 have_heard_from_all = __get_heard_from_all_status(priv);
6377 if (!had_heard_from_all && have_heard_from_all) {
6378 if (priv->timer) {
6379 gf_timer_call_cancel(this->ctx, priv->timer);
6380 priv->timer = NULL;
6381 }
6382 /* This is the first event which completes aggregation
6383 of events from all subvolumes. If at least one subvol
6384 had come up, propagate CHILD_UP, but only this time
6385 */
6386 event = GF_EVENT_CHILD_DOWN;
6387 for (i = 0; i < priv->child_count; i++) {
6388 if (priv->last_event[i] == GF_EVENT_CHILD_UP) {
6389 event = GF_EVENT_CHILD_UP;
6390 break;
6391 }
6392
6393 if (priv->last_event[i] == GF_EVENT_CHILD_CONNECTING) {
6394 event = GF_EVENT_CHILD_CONNECTING;
6395 /* continue to check other events for CHILD_UP */
6396 }
6397 }
6398 }
6399 }
6400 UNLOCK(&priv->lock);
6401
6402 if (priv->quorum_count) {
6403 has_quorum = afr_has_quorum(priv->child_up, priv, NULL);
6404 if (!had_quorum && has_quorum) {
6405 gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_QUORUM_MET,
6406 "Client-quorum is met");
6407 gf_event(EVENT_AFR_QUORUM_MET, "client-pid=%d; subvol=%s",
6408 this->ctx->cmd_args.client_pid, this->name);
6409 }
6410 if (had_quorum && !has_quorum) {
6411 gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_QUORUM_FAIL,
6412 "Client-quorum is not met");
6413 gf_event(EVENT_AFR_QUORUM_FAIL, "client-pid=%d; subvol=%s",
6414 this->ctx->cmd_args.client_pid, this->name);
6415 }
6416 }
6417
6418 /* if all subvols have reported status, no need to hide anything
6419 or wait for anything else. Just propagate blindly */
6420 if (have_heard_from_all)
6421 propagate = 1;
6422
6423 ret = 0;
6424 if (propagate)
6425 ret = default_notify(this, event, data);
6426
6427 if ((!had_heard_from_all) || call_psh) {
6428 /* Launch self-heal on all local subvolumes if:
6429 * a) We have_heard_from_all for the first time
6430 * b) Already heard from everyone, but we now got a child-up
6431 * event.
6432 */
6433 if (have_heard_from_all) {
6434 afr_selfheal_childup(this, priv);
6435 }
6436 }
6437 out:
6438 return ret;
6439 }
6440
6441 int
6442 afr_local_init(afr_local_t *local, afr_private_t *priv, int32_t *op_errno)
6443 {
6444 int __ret;
6445 local->op_ret = -1;
6446 local->op_errno = EUCLEAN;
6447
6448 __ret = syncbarrier_init(&local->barrier);
6449 if (__ret) {
6450 if (op_errno)
6451 *op_errno = __ret;
6452 goto out;
6453 }
6454
6455 local->child_up = GF_MALLOC(priv->child_count * sizeof(*local->child_up),
6456 gf_afr_mt_char);
6457 if (!local->child_up) {
6458 if (op_errno)
6459 *op_errno = ENOMEM;
6460 goto out;
6461 }
6462
6463 memcpy(local->child_up, priv->child_up,
6464 sizeof(*local->child_up) * priv->child_count);
6465 local->call_count = AFR_COUNT(local->child_up, priv->child_count);
6466 if (local->call_count == 0) {
6467 gf_msg(THIS->name, GF_LOG_INFO, 0, AFR_MSG_SUBVOLS_DOWN,
6468 "no subvolumes up");
6469 if (op_errno)
6470 *op_errno = ENOTCONN;
6471 goto out;
6472 }
6473
6474 local->event_generation = priv->event_generation;
6475
6476 local->read_attempted = GF_CALLOC(priv->child_count, sizeof(char),
6477 gf_afr_mt_char);
6478 if (!local->read_attempted) {
6479 if (op_errno)
6480 *op_errno = ENOMEM;
6481 goto out;
6482 }
6483
6484 local->readable = GF_CALLOC(priv->child_count, sizeof(char),
6485 gf_afr_mt_char);
6486 if (!local->readable) {
6487 if (op_errno)
6488 *op_errno = ENOMEM;
6489 goto out;
6490 }
6491
6492 local->readable2 = GF_CALLOC(priv->child_count, sizeof(char),
6493 gf_afr_mt_char);
6494 if (!local->readable2) {
6495 if (op_errno)
6496 *op_errno = ENOMEM;
6497 goto out;
6498 }
6499
6500 local->read_subvol = -1;
6501
6502 local->replies = GF_CALLOC(priv->child_count, sizeof(*local->replies),
6503 gf_afr_mt_reply_t);
6504 if (!local->replies) {
6505 if (op_errno)
6506 *op_errno = ENOMEM;
6507 goto out;
6508 }
6509
6510 local->need_full_crawl = _gf_false;
6511 if (priv->thin_arbiter_count) {
6512 local->ta_child_up = priv->ta_child_up;
6513 local->ta_failed_subvol = AFR_CHILD_UNKNOWN;
6514 local->read_txn_query_child = AFR_CHILD_UNKNOWN;
6515 local->ta_event_gen = priv->ta_event_gen;
6516 local->fop_state = TA_SUCCESS;
6517 }
6518 local->is_new_entry = _gf_false;
6519
6520 local->need_open = GF_CALLOC(priv->child_count, sizeof(*local->need_open),
6521 gf_afr_mt_char);
6522 if (!local->need_open) {
6523 if (op_errno)
6524 *op_errno = ENOMEM;
6525 goto out;
6526 }
6527
6528 INIT_LIST_HEAD(&local->healer);
6529 return 0;
6530 out:
6531 return -1;
6532 }
6533
6534 static inline void
6535 afr_internal_lock_init(afr_internal_lock_t *lk, size_t child_count)
6536 {
6537 lk->lock_op_ret = -1;
6538 lk->lock_op_errno = EUCLEAN;
6539 }
6540
6541 void
6542 afr_matrix_cleanup(int32_t **matrix, unsigned int m)
6543 {
6544 int i = 0;
6545
6546 if (!matrix)
6547 goto out;
6548 for (i = 0; i < m; i++) {
6549 GF_FREE(matrix[i]);
6550 }
6551
6552 GF_FREE(matrix);
6553 out:
6554 return;
6555 }
6556
6557 int32_t **
6558 afr_matrix_create(unsigned int m, unsigned int n)
6559 {
6560 int32_t **matrix = NULL;
6561 int i = 0;
6562
6563 matrix = GF_CALLOC(sizeof(*matrix), m, gf_afr_mt_int32_t);
6564 if (!matrix)
6565 goto out;
6566
6567 for (i = 0; i < m; i++) {
6568 matrix[i] = GF_CALLOC(sizeof(*matrix[i]), n, gf_afr_mt_int32_t);
6569 if (!matrix[i])
6570 goto out;
6571 }
6572 return matrix;
6573 out:
6574 afr_matrix_cleanup(matrix, m);
6575 return NULL;
6576 }
6577
6578 int
6579 afr_transaction_local_init(afr_local_t *local, xlator_t *this)
6580 {
6581 int ret = -ENOMEM;
6582 afr_private_t *priv = NULL;
6583
6584 priv = this->private;
6585 INIT_LIST_HEAD(&local->transaction.wait_list);
6586 INIT_LIST_HEAD(&local->transaction.owner_list);
6587 INIT_LIST_HEAD(&local->ta_waitq);
6588 INIT_LIST_HEAD(&local->ta_onwireq);
6589 afr_internal_lock_init(&local->internal_lock, priv->child_count);
6590
6591 ret = -ENOMEM;
6592 local->pre_op_compat = priv->pre_op_compat;
6593
6594 local->transaction.pre_op = GF_CALLOC(sizeof(*local->transaction.pre_op),
6595 priv->child_count, gf_afr_mt_char);
6596 if (!local->transaction.pre_op)
6597 goto out;
6598
6599 local->transaction.changelog_xdata = GF_CALLOC(
6600 sizeof(*local->transaction.changelog_xdata), priv->child_count,
6601 gf_afr_mt_dict_t);
6602 if (!local->transaction.changelog_xdata)
6603 goto out;
6604
6605 if (priv->arbiter_count == 1) {
6606 local->transaction.pre_op_sources = GF_CALLOC(
6607 sizeof(*local->transaction.pre_op_sources), priv->child_count,
6608 gf_afr_mt_char);
6609 if (!local->transaction.pre_op_sources)
6610 goto out;
6611 }
6612
6613 local->transaction.failed_subvols = GF_CALLOC(
6614 sizeof(*local->transaction.failed_subvols), priv->child_count,
6615 gf_afr_mt_char);
6616 if (!local->transaction.failed_subvols)
6617 goto out;
6618
6619 local->pending = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
6620 if (!local->pending)
6621 goto out;
6622
6623 ret = 0;
6624 out:
6625 return ret;
6626 }
6627
6628 void
6629 afr_priv_destroy(afr_private_t *priv)
6630 {
6631 int i = 0;
6632 int child_count = -1;
6633
6634 if (!priv)
6635 goto out;
6636
6637 GF_FREE(priv->sh_domain);
6638 GF_FREE(priv->last_event);
6639
6640 child_count = priv->child_count;
6641 if (priv->thin_arbiter_count) {
6642 child_count++;
6643 }
6644 if (priv->pending_key) {
6645 for (i = 0; i < child_count; i++)
6646 GF_FREE(priv->pending_key[i]);
6647 }
6648
6649 GF_FREE(priv->pending_reads);
6650 GF_FREE(priv->local);
6651 GF_FREE(priv->pending_key);
6652 GF_FREE(priv->children);
6653 GF_FREE(priv->anon_inode);
6654 GF_FREE(priv->child_up);
6655 GF_FREE(priv->halo_child_up);
6656 GF_FREE(priv->child_latency);
6657 LOCK_DESTROY(&priv->lock);
6658
6659 GF_FREE(priv);
6660 out:
6661 return;
6662 }
6663
6664 int **
6665 afr_mark_pending_changelog(afr_private_t *priv, unsigned char *pending,
6666 dict_t *xattr, ia_type_t iat)
6667 {
6668 int i = 0;
6669 int **changelog = NULL;
6670 int idx = -1;
6671 int m_idx = 0;
6672 int d_idx = 0;
6673 int ret = 0;
6674 uint32_t hton32_1;
6675
6676 m_idx = afr_index_for_transaction_type(AFR_METADATA_TRANSACTION);
6677 d_idx = afr_index_for_transaction_type(AFR_DATA_TRANSACTION);
6678
6679 idx = afr_index_from_ia_type(iat);
6680
6681 changelog = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
6682 if (!changelog)
6683 goto out;
6684
6685 hton32_1 = htobe32(1);
6686 for (i = 0; i < priv->child_count; i++) {
6687 if (!pending[i])
6688 continue;
6689
6690 changelog[i][m_idx] = hton32_1;
6691 if (idx != -1)
6692 changelog[i][idx] = hton32_1;
6693 /* If the newentry marking is on a newly created directory,
6694 * then mark it with the full-heal indicator.
6695 */
6696 if ((IA_ISDIR(iat)) && (priv->esh_granular))
6697 changelog[i][d_idx] = hton32_1;
6698 }
6699 ret = afr_set_pending_dict(priv, xattr, changelog);
6700 if (ret < 0) {
6701 afr_matrix_cleanup(changelog, priv->child_count);
6702 return NULL;
6703 }
6704 out:
6705 return changelog;
6706 }
6707
6708 static dict_t *
6709 afr_set_heal_info(char *status)
6710 {
6711 dict_t *dict = NULL;
6712 int ret = -1;
6713
6714 dict = dict_new();
6715 if (!dict) {
6716 ret = -ENOMEM;
6717 goto out;
6718 }
6719
6720 ret = dict_set_dynstr_sizen(dict, "heal-info", status);
6721 if (ret)
6722 gf_msg("", GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
6723 "Failed to set heal-info key to "
6724 "%s",
6725 status);
6726 out:
6727 /* Any error other than EINVAL, dict_set_dynstr frees status */
6728 if (ret == -ENOMEM || ret == -EINVAL) {
6729 GF_FREE(status);
6730 }
6731
6732 if (ret && dict) {
6733 dict_unref(dict);
6734 dict = NULL;
6735 }
6736 return dict;
6737 }
6738
6739 static gf_boolean_t
6740 afr_is_dirty_count_non_unary_for_txn(xlator_t *this, struct afr_reply *replies,
6741 afr_transaction_type type)
6742 {
6743 afr_private_t *priv = this->private;
6744 int *dirty = alloca0(priv->child_count * sizeof(int));
6745 int i = 0;
6746
6747 afr_selfheal_extract_xattr(this, replies, type, dirty, NULL);
6748 for (i = 0; i < priv->child_count; i++) {
6749 if (dirty[i] > 1)
6750 return _gf_true;
6751 }
6752
6753 return _gf_false;
6754 }
6755
6756 static gf_boolean_t
6757 afr_is_dirty_count_non_unary(xlator_t *this, struct afr_reply *replies,
6758 ia_type_t ia_type)
6759 {
6760 gf_boolean_t data_chk = _gf_false;
6761 gf_boolean_t mdata_chk = _gf_false;
6762 gf_boolean_t entry_chk = _gf_false;
6763
6764 switch (ia_type) {
6765 case IA_IFDIR:
6766 mdata_chk = _gf_true;
6767 entry_chk = _gf_true;
6768 break;
6769 case IA_IFREG:
6770 mdata_chk = _gf_true;
6771 data_chk = _gf_true;
6772 break;
6773 default:
6774 /*IA_IFBLK, IA_IFCHR, IA_IFLNK, IA_IFIFO, IA_IFSOCK*/
6775 mdata_chk = _gf_true;
6776 break;
6777 }
6778
6779 if (data_chk && afr_is_dirty_count_non_unary_for_txn(
6780 this, replies, AFR_DATA_TRANSACTION)) {
6781 return _gf_true;
6782 } else if (mdata_chk && afr_is_dirty_count_non_unary_for_txn(
6783 this, replies, AFR_METADATA_TRANSACTION)) {
6784 return _gf_true;
6785 } else if (entry_chk && afr_is_dirty_count_non_unary_for_txn(
6786 this, replies, AFR_ENTRY_TRANSACTION)) {
6787 return _gf_true;
6788 }
6789
6790 return _gf_false;
6791 }
6792
6793 static int
6794 afr_update_heal_status(xlator_t *this, struct afr_reply *replies,
6795 ia_type_t ia_type, gf_boolean_t *esh, gf_boolean_t *dsh,
6796 gf_boolean_t *msh, unsigned char pending)
6797 {
6798 int ret = -1;
6799 GF_UNUSED int ret1 = 0;
6800 int i = 0;
6801 int io_domain_lk_count = 0;
6802 int shd_domain_lk_count = 0;
6803 afr_private_t *priv = NULL;
6804 char *key1 = NULL;
6805 char *key2 = NULL;
6806
6807 priv = this->private;
6808 key1 = alloca0(strlen(GLUSTERFS_INODELK_DOM_PREFIX) + 2 +
6809 strlen(this->name));
6810 key2 = alloca0(strlen(GLUSTERFS_INODELK_DOM_PREFIX) + 2 +
6811 strlen(priv->sh_domain));
6812 sprintf(key1, "%s:%s", GLUSTERFS_INODELK_DOM_PREFIX, this->name);
6813 sprintf(key2, "%s:%s", GLUSTERFS_INODELK_DOM_PREFIX, priv->sh_domain);
6814
6815 for (i = 0; i < priv->child_count; i++) {
6816 if ((replies[i].valid != 1) || (replies[i].op_ret != 0))
6817 continue;
6818 if (!io_domain_lk_count) {
6819 ret1 = dict_get_int32(replies[i].xdata, key1, &io_domain_lk_count);
6820 }
6821 if (!shd_domain_lk_count) {
6822 ret1 = dict_get_int32(replies[i].xdata, key2, &shd_domain_lk_count);
6823 }
6824 }
6825
6826 if (!pending) {
6827 if ((afr_is_dirty_count_non_unary(this, replies, ia_type)) ||
6828 (!io_domain_lk_count)) {
6829 /* Needs heal. */
6830 ret = 0;
6831 } else {
6832 /* No heal needed. */
6833 *dsh = *esh = *msh = 0;
6834 }
6835 } else {
6836 if (shd_domain_lk_count) {
6837 ret = -EAGAIN; /*For 'possibly-healing'. */
6838 } else {
6839 ret = 0; /*needs heal. Just set a non -ve value so that it is
6840 assumed as the source index.*/
6841 }
6842 }
6843 return ret;
6844 }
6845
6846 /*return EIO, EAGAIN or pending*/
6847 static int
6848 afr_lockless_inspect(call_frame_t *frame, xlator_t *this, uuid_t gfid,
6849 inode_t **inode, gf_boolean_t *entry_selfheal,
6850 gf_boolean_t *data_selfheal,
6851 gf_boolean_t *metadata_selfheal, unsigned char *pending)
6852 {
6853 int ret = -1;
6854 int i = 0;
6855 afr_private_t *priv = NULL;
6856 struct afr_reply *replies = NULL;
6857 gf_boolean_t dsh = _gf_false;
6858 gf_boolean_t msh = _gf_false;
6859 gf_boolean_t esh = _gf_false;
6860 unsigned char *sources = NULL;
6861 unsigned char *sinks = NULL;
6862 unsigned char *valid_on = NULL;
6863 uint64_t *witness = NULL;
6864
6865 priv = this->private;
6866 replies = alloca0(sizeof(*replies) * priv->child_count);
6867 sources = alloca0(sizeof(*sources) * priv->child_count);
6868 sinks = alloca0(sizeof(*sinks) * priv->child_count);
6869 witness = alloca0(sizeof(*witness) * priv->child_count);
6870 valid_on = alloca0(sizeof(*valid_on) * priv->child_count);
6871
6872 ret = afr_selfheal_unlocked_inspect(frame, this, gfid, inode, &dsh, &msh,
6873 &esh, replies);
6874 if (ret)
6875 goto out;
6876 for (i = 0; i < priv->child_count; i++) {
6877 if (replies[i].valid && replies[i].op_ret == 0) {
6878 valid_on[i] = 1;
6879 }
6880 }
6881 if (msh) {
6882 ret = afr_selfheal_find_direction(frame, this, replies,
6883 AFR_METADATA_TRANSACTION, valid_on,
6884 sources, sinks, witness, pending);
6885 if (*pending & PFLAG_SBRAIN)
6886 ret = -EIO;
6887 if (ret)
6888 goto out;
6889 }
6890 if (dsh) {
6891 ret = afr_selfheal_find_direction(frame, this, replies,
6892 AFR_DATA_TRANSACTION, valid_on,
6893 sources, sinks, witness, pending);
6894 if (*pending & PFLAG_SBRAIN)
6895 ret = -EIO;
6896 if (ret)
6897 goto out;
6898 }
6899 if (esh) {
6900 ret = afr_selfheal_find_direction(frame, this, replies,
6901 AFR_ENTRY_TRANSACTION, valid_on,
6902 sources, sinks, witness, pending);
6903 if (*pending & PFLAG_SBRAIN)
6904 ret = -EIO;
6905 if (ret)
6906 goto out;
6907 }
6908
6909 ret = afr_update_heal_status(this, replies, (*inode)->ia_type, &esh, &dsh,
6910 &msh, *pending);
6911 out:
6912 *data_selfheal = dsh;
6913 *entry_selfheal = esh;
6914 *metadata_selfheal = msh;
6915 if (replies)
6916 afr_replies_wipe(replies, priv->child_count);
6917 return ret;
6918 }
6919
6920 int
6921 afr_get_heal_info(call_frame_t *frame, xlator_t *this, loc_t *loc)
6922 {
6923 gf_boolean_t data_selfheal = _gf_false;
6924 gf_boolean_t metadata_selfheal = _gf_false;
6925 gf_boolean_t entry_selfheal = _gf_false;
6926 unsigned char pending = 0;
6927 dict_t *dict = NULL;
6928 int ret = -1;
6929 int op_errno = ENOMEM;
6930 inode_t *inode = NULL;
6931 char *substr = NULL;
6932 char *status = NULL;
6933 call_frame_t *heal_frame = NULL;
6934 afr_local_t *heal_local = NULL;
6935
6936 /*Use frame with lk-owner set*/
6937 heal_frame = afr_frame_create(frame->this, &op_errno);
6938 if (!heal_frame) {
6939 ret = -1;
6940 goto out;
6941 }
6942 heal_local = heal_frame->local;
6943 heal_frame->local = frame->local;
6944
6945 ret = afr_lockless_inspect(heal_frame, this, loc->gfid, &inode,
6946 &entry_selfheal, &data_selfheal,
6947 &metadata_selfheal, &pending);
6948
6949 if (ret == -ENOMEM) {
6950 ret = -1;
6951 goto out;
6952 }
6953
6954 if (pending & PFLAG_PENDING) {
6955 gf_asprintf(&substr, "-pending");
6956 if (!substr)
6957 goto out;
6958 }
6959
6960 if (ret == -EIO) {
6961 ret = gf_asprintf(&status, "split-brain%s", substr ? substr : "");
6962 if (ret < 0) {
6963 goto out;
6964 }
6965 dict = afr_set_heal_info(status);
6966 if (!dict) {
6967 ret = -1;
6968 goto out;
6969 }
6970 } else if (ret == -EAGAIN) {
6971 ret = gf_asprintf(&status, "possibly-healing%s", substr ? substr : "");
6972 if (ret < 0) {
6973 goto out;
6974 }
6975 dict = afr_set_heal_info(status);
6976 if (!dict) {
6977 ret = -1;
6978 goto out;
6979 }
6980 } else if (ret >= 0) {
6981 /* value of ret = source index
6982 * so ret >= 0 and at least one of the 3 booleans set to
6983 * true means a source is identified; heal is required.
6984 */
6985 if (!data_selfheal && !entry_selfheal && !metadata_selfheal) {
6986 status = gf_strdup("no-heal");
6987 if (!status) {
6988 ret = -1;
6989 goto out;
6990 }
6991 dict = afr_set_heal_info(status);
6992 if (!dict) {
6993 ret = -1;
6994 goto out;
6995 }
6996 } else {
6997 ret = gf_asprintf(&status, "heal%s", substr ? substr : "");
6998 if (ret < 0) {
6999 goto out;
7000 }
7001 dict = afr_set_heal_info(status);
7002 if (!dict) {
7003 ret = -1;
7004 goto out;
7005 }
7006 }
7007 } else if (ret < 0) {
7008 /* Apart from above checked -ve ret values, there are
7009 * other possible ret values like ENOTCONN
7010 * (returned when number of valid replies received are
7011 * less than 2)
7012 * in which case heal is required when one of the
7013 * selfheal booleans is set.
7014 */
7015 if (data_selfheal || entry_selfheal || metadata_selfheal) {
7016 ret = gf_asprintf(&status, "heal%s", substr ? substr : "");
7017 if (ret < 0) {
7018 goto out;
7019 }
7020 dict = afr_set_heal_info(status);
7021 if (!dict) {
7022 ret = -1;
7023 goto out;
7024 }
7025 }
7026 }
7027
7028 ret = 0;
7029 op_errno = 0;
7030
7031 out:
7032 if (heal_frame) {
7033 heal_frame->local = heal_local;
7034 AFR_STACK_DESTROY(heal_frame);
7035 }
7036 AFR_STACK_UNWIND(getxattr, frame, ret, op_errno, dict, NULL);
7037 if (dict)
7038 dict_unref(dict);
7039 if (inode)
7040 inode_unref(inode);
7041 GF_FREE(substr);
7042 return ret;
7043 }
7044
7045 int
7046 _afr_is_split_brain(call_frame_t *frame, xlator_t *this,
7047 struct afr_reply *replies, afr_transaction_type type,
7048 gf_boolean_t *spb)
7049 {
7050 afr_private_t *priv = NULL;
7051 uint64_t *witness = NULL;
7052 unsigned char *sources = NULL;
7053 unsigned char *sinks = NULL;
7054 int sources_count = 0;
7055 int ret = 0;
7056
7057 priv = this->private;
7058
7059 sources = alloca0(priv->child_count);
7060 sinks = alloca0(priv->child_count);
7061 witness = alloca0(priv->child_count * sizeof(*witness));
7062
7063 ret = afr_selfheal_find_direction(frame, this, replies, type,
7064 priv->child_up, sources, sinks, witness,
7065 NULL);
7066 if (ret)
7067 return ret;
7068
7069 sources_count = AFR_COUNT(sources, priv->child_count);
7070 if (!sources_count)
7071 *spb = _gf_true;
7072
7073 return ret;
7074 }
7075
7076 int
7077 afr_is_split_brain(call_frame_t *frame, xlator_t *this, inode_t *inode,
7078 uuid_t gfid, gf_boolean_t *d_spb, gf_boolean_t *m_spb)
7079 {
7080 int ret = -1;
7081 afr_private_t *priv = NULL;
7082 struct afr_reply *replies = NULL;
7083
7084 priv = this->private;
7085
7086 replies = alloca0(sizeof(*replies) * priv->child_count);
7087
7088 ret = afr_selfheal_unlocked_discover(frame, inode, gfid, replies);
7089 if (ret)
7090 goto out;
7091
7092 if (!afr_can_decide_split_brain_source_sinks(replies, priv->child_count)) {
7093 ret = -EAGAIN;
7094 goto out;
7095 }
7096
7097 ret = _afr_is_split_brain(frame, this, replies, AFR_DATA_TRANSACTION,
7098 d_spb);
7099 if (ret)
7100 goto out;
7101
7102 ret = _afr_is_split_brain(frame, this, replies, AFR_METADATA_TRANSACTION,
7103 m_spb);
7104 out:
7105 if (replies) {
7106 afr_replies_wipe(replies, priv->child_count);
7107 replies = NULL;
7108 }
7109 return ret;
7110 }
7111
7112 int
7113 afr_get_split_brain_status_cbk(int ret, call_frame_t *frame, void *opaque)
7114 {
7115 GF_FREE(opaque);
7116 return 0;
7117 }
7118
7119 int
7120 afr_get_split_brain_status(void *opaque)
7121 {
7122 gf_boolean_t d_spb = _gf_false;
7123 gf_boolean_t m_spb = _gf_false;
7124 int ret = -1;
7125 int op_errno = 0;
7126 int i = 0;
7127 char *choices = NULL;
7128 char *status = NULL;
7129 dict_t *dict = NULL;
7130 inode_t *inode = NULL;
7131 afr_private_t *priv = NULL;
7132 xlator_t **children = NULL;
7133 call_frame_t *frame = NULL;
7134 xlator_t *this = NULL;
7135 loc_t *loc = NULL;
7136 afr_spb_status_t *data = NULL;
7137
7138 data = opaque;
7139 frame = data->frame;
7140 this = frame->this;
7141 loc = data->loc;
7142 priv = this->private;
7143 children = priv->children;
7144
7145 inode = afr_inode_find(this, loc->gfid);
7146 if (!inode)
7147 goto out;
7148
7149 dict = dict_new();
7150 if (!dict) {
7151 op_errno = ENOMEM;
7152 ret = -1;
7153 goto out;
7154 }
7155
7156 /* Calculation for string length :
7157 * (child_count X length of child-name) + SLEN(" Choices :")
7158 * child-name consists of :
7159 * a) 251 = max characters for volname according to GD_VOLUME_NAME_MAX
7160 * b) strlen("-client-00,") assuming 16 replicas
7161 */
7162 choices = alloca0(priv->child_count * (256 + SLEN("-client-00,")) +
7163 SLEN(" Choices:"));
7164
7165 ret = afr_is_split_brain(frame, this, inode, loc->gfid, &d_spb, &m_spb);
7166 if (ret) {
7167 op_errno = -ret;
7168 if (ret == -EAGAIN) {
7169 ret = dict_set_sizen_str_sizen(dict, GF_AFR_SBRAIN_STATUS,
7170 SBRAIN_HEAL_NO_GO_MSG);
7171 if (ret) {
7172 gf_msg(this->name, GF_LOG_WARNING, -ret,
7173 AFR_MSG_DICT_SET_FAILED,
7174 "Failed to set GF_AFR_SBRAIN_STATUS in dict");
7175 }
7176 }
7177 ret = -1;
7178 goto out;
7179 }
7180
7181 if (d_spb || m_spb) {
7182 sprintf(choices, " Choices:");
7183 for (i = 0; i < priv->child_count; i++) {
7184 strcat(choices, children[i]->name);
7185 strcat(choices, ",");
7186 }
7187 choices[strlen(choices) - 1] = '\0';
7188
7189 ret = gf_asprintf(&status,
7190 "data-split-brain:%s "
7191 "metadata-split-brain:%s%s",
7192 (d_spb) ? "yes" : "no", (m_spb) ? "yes" : "no",
7193 choices);
7194
7195 if (-1 == ret) {
7196 op_errno = ENOMEM;
7197 goto out;
7198 }
7199 ret = dict_set_dynstr_sizen(dict, GF_AFR_SBRAIN_STATUS, status);
7200 if (ret) {
7201 op_errno = -ret;
7202 ret = -1;
7203 goto out;
7204 }
7205 } else {
7206 ret = dict_set_sizen_str_sizen(dict, GF_AFR_SBRAIN_STATUS,
7207 SFILE_NOT_UNDER_DATA);
7208 if (ret) {
7209 op_errno = -ret;
7210 ret = -1;
7211 goto out;
7212 }
7213 }
7214
7215 ret = 0;
7216 out:
7217 AFR_STACK_UNWIND(getxattr, frame, ret, op_errno, dict, NULL);
7218 if (dict)
7219 dict_unref(dict);
7220 if (inode)
7221 inode_unref(inode);
7222 return ret;
7223 }
7224
7225 int32_t
7226 afr_heal_splitbrain_file(call_frame_t *frame, xlator_t *this, loc_t *loc)
7227 {
7228 int ret = 0;
7229 int op_errno = 0;
7230 dict_t *dict = NULL;
7231 afr_local_t *local = NULL;
7232 afr_local_t *heal_local = NULL;
7233 call_frame_t *heal_frame = NULL;
7234
7235 local = frame->local;
7236 dict = dict_new();
7237 if (!dict) {
7238 op_errno = ENOMEM;
7239 ret = -1;
7240 goto out;
7241 }
7242
7243 heal_frame = afr_frame_create(this, &op_errno);
7244 if (!heal_frame) {
7245 ret = -1;
7246 goto out;
7247 }
7248 heal_local = heal_frame->local;
7249 heal_frame->local = frame->local;
7250 /*Initiate heal with heal_frame with lk-owner set so that inodelk/entrylk
7251 * work correctly*/
7252 ret = afr_selfheal_do(heal_frame, this, loc->gfid);
7253
7254 if (ret == 1 || ret == 2) {
7255 ret = dict_set_sizen_str_sizen(dict, "sh-fail-msg",
7256 SFILE_NOT_IN_SPLIT_BRAIN);
7257 if (ret)
7258 gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
7259 "Failed to set sh-fail-msg in dict");
7260 ret = 0;
7261 goto out;
7262 } else {
7263 if (local->xdata_rsp) {
7264 /* 'sh-fail-msg' has been set in the dict during self-heal.*/
7265 dict_copy(local->xdata_rsp, dict);
7266 ret = 0;
7267 } else if (ret < 0) {
7268 op_errno = -ret;
7269 ret = -1;
7270 }
7271 }
7272
7273 out:
7274 if (heal_frame) {
7275 heal_frame->local = heal_local;
7276 AFR_STACK_DESTROY(heal_frame);
7277 }
7278 if (local->op == GF_FOP_GETXATTR)
7279 AFR_STACK_UNWIND(getxattr, frame, ret, op_errno, dict, NULL);
7280 else if (local->op == GF_FOP_SETXATTR)
7281 AFR_STACK_UNWIND(setxattr, frame, ret, op_errno, NULL);
7282 if (dict)
7283 dict_unref(dict);
7284 return ret;
7285 }
7286
7287 int
7288 afr_get_child_index_from_name(xlator_t *this, char *name)
7289 {
7290 afr_private_t *priv = this->private;
7291 int index = -1;
7292
7293 for (index = 0; index < priv->child_count; index++) {
7294 if (!strcmp(priv->children[index]->name, name))
7295 goto out;
7296 }
7297 index = -1;
7298 out:
7299 return index;
7300 }
7301
7302 void
7303 afr_priv_need_heal_set(afr_private_t *priv, gf_boolean_t need_heal)
7304 {
7305 LOCK(&priv->lock);
7306 {
7307 priv->need_heal = need_heal;
7308 }
7309 UNLOCK(&priv->lock);
7310 }
7311
7312 static void
7313 afr_set_need_heal(xlator_t *this, afr_local_t *local)
7314 {
7315 int i = 0;
7316 afr_private_t *priv = this->private;
7317 gf_boolean_t need_heal = _gf_false;
7318
7319 for (i = 0; i < priv->child_count; i++) {
7320 if (local->replies[i].valid && local->replies[i].need_heal) {
7321 need_heal = _gf_true;
7322 break;
7323 }
7324 }
7325 afr_priv_need_heal_set(priv, need_heal);
7326 return;
7327 }
7328
7329 gf_boolean_t
7330 afr_get_need_heal(afr_private_t *priv)
7331 {
7332 gf_boolean_t need_heal = _gf_true;
7333
7334 LOCK(&priv->lock);
7335 {
7336 need_heal = priv->need_heal;
7337 }
7338 UNLOCK(&priv->lock);
7339 return need_heal;
7340 }
7341
7342 static int
7343 afr_fav_child_reset_sink_xattrs_cbk(int ret, call_frame_t *heal_frame,
7344 void *opaque)
7345 {
7346 call_frame_t *txn_frame = NULL;
7347 afr_local_t *local = NULL;
7348 afr_local_t *heal_local = NULL;
7349 xlator_t *this = NULL;
7350
7351 heal_local = heal_frame->local;
7352 txn_frame = heal_local->heal_frame;
7353 local = txn_frame->local;
7354 this = txn_frame->this;
7355
7356 /* Refresh the inode agan and proceed with the transaction.*/
7357 afr_inode_refresh(txn_frame, this, local->inode, NULL, local->refreshfn);
7358
7359 AFR_STACK_DESTROY(heal_frame);
7360
7361 return 0;
7362 }
7363
7364 int
7365 afr_fav_child_reset_sink_xattrs(void *opaque)
7366 {
7367 call_frame_t *heal_frame = NULL;
7368 call_frame_t *txn_frame = NULL;
7369 xlator_t *this = NULL;
7370 gf_boolean_t d_spb = _gf_false;
7371 gf_boolean_t m_spb = _gf_false;
7372 afr_local_t *heal_local = NULL;
7373 afr_local_t *txn_local = NULL;
7374 afr_private_t *priv = NULL;
7375 inode_t *inode = NULL;
7376 unsigned char *locked_on = NULL;
7377 unsigned char *sources = NULL;
7378 unsigned char *sinks = NULL;
7379 unsigned char *healed_sinks = NULL;
7380 unsigned char *undid_pending = NULL;
7381 struct afr_reply *locked_replies = NULL;
7382 int ret = 0;
7383
7384 heal_frame = (call_frame_t *)opaque;
7385 heal_local = heal_frame->local;
7386 txn_frame = heal_local->heal_frame;
7387 txn_local = txn_frame->local;
7388 this = txn_frame->this;
7389 inode = txn_local->inode;
7390 priv = this->private;
7391 locked_on = alloca0(priv->child_count);
7392 sources = alloca0(priv->child_count);
7393 sinks = alloca0(priv->child_count);
7394 healed_sinks = alloca0(priv->child_count);
7395 undid_pending = alloca0(priv->child_count);
7396 locked_replies = alloca0(sizeof(*locked_replies) * priv->child_count);
7397
7398 ret = _afr_is_split_brain(txn_frame, this, txn_local->replies,
7399 AFR_DATA_TRANSACTION, &d_spb);
7400
7401 ret = _afr_is_split_brain(txn_frame, this, txn_local->replies,
7402 AFR_METADATA_TRANSACTION, &m_spb);
7403
7404 /* Take appropriate locks and reset sink xattrs. */
7405 if (d_spb) {
7406 ret = afr_selfheal_inodelk(heal_frame, this, inode, this->name, 0, 0,
7407 locked_on);
7408 {
7409 if (ret < priv->child_count)
7410 goto data_unlock;
7411 ret = __afr_selfheal_data_prepare(
7412 heal_frame, this, inode, locked_on, sources, sinks,
7413 healed_sinks, undid_pending, locked_replies, NULL);
7414 }
7415 data_unlock:
7416 afr_selfheal_uninodelk(heal_frame, this, inode, this->name, 0, 0,
7417 locked_on);
7418 }
7419
7420 if (m_spb) {
7421 memset(locked_on, 0, sizeof(*locked_on) * priv->child_count);
7422 memset(undid_pending, 0, sizeof(*undid_pending) * priv->child_count);
7423 ret = afr_selfheal_inodelk(heal_frame, this, inode, this->name,
7424 LLONG_MAX - 1, 0, locked_on);
7425 {
7426 if (ret < priv->child_count)
7427 goto mdata_unlock;
7428 ret = __afr_selfheal_metadata_prepare(
7429 heal_frame, this, inode, locked_on, sources, sinks,
7430 healed_sinks, undid_pending, locked_replies, NULL);
7431 }
7432 mdata_unlock:
7433 afr_selfheal_uninodelk(heal_frame, this, inode, this->name,
7434 LLONG_MAX - 1, 0, locked_on);
7435 }
7436
7437 return ret;
7438 }
7439
7440 /*
7441 * Concatenates the xattrs in local->replies separated by a delimiter.
7442 */
7443 int
7444 afr_serialize_xattrs_with_delimiter(call_frame_t *frame, xlator_t *this,
7445 char *buf, size_t size,
7446 const char *default_str, int32_t *serz_len,
7447 char delimiter)
7448 {
7449 afr_private_t *priv = NULL;
7450 afr_local_t *local = NULL;
7451 char *xattr = NULL;
7452 int i = 0;
7453 size_t len = 0;
7454 int keylen = 0;
7455 int ret = -1;
7456
7457 priv = this->private;
7458 local = frame->local;
7459
7460 keylen = strlen(local->cont.getxattr.name);
7461 for (i = 0; i < priv->child_count; i++) {
7462 if (!local->replies[i].valid || local->replies[i].op_ret) {
7463 len += snprintf(buf + len, size - len, "%s%c", default_str,
7464 delimiter);
7465 } else {
7466 ret = dict_get_strn(local->replies[i].xattr,
7467 local->cont.getxattr.name, keylen, &xattr);
7468 if (ret) {
7469 gf_msg("TEST", GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
7470 "Failed to get the node_uuid of brick "
7471 "%d",
7472 i);
7473 goto out;
7474 }
7475 len += snprintf(buf + len, size - len, "%s%c", xattr, delimiter);
7476 }
7477 /* No overflow but buffer size is not enough. */
7478 GF_ASSERT(len <= size);
7479 }
7480 buf[--len] = '\0'; /*remove the last delimiter*/
7481 if (serz_len)
7482 *serz_len = ++len;
7483 ret = 0;
7484
7485 out:
7486 return ret;
7487 }
7488
7489 uint64_t
7490 afr_write_subvol_get(call_frame_t *frame, xlator_t *this)
7491 {
7492 afr_local_t *local = NULL;
7493 uint64_t write_subvol = 0;
7494
7495 local = frame->local;
7496 LOCK(&local->inode->lock);
7497 write_subvol = local->inode_ctx->write_subvol;
7498 UNLOCK(&local->inode->lock);
7499
7500 return write_subvol;
7501 }
7502
7503 int
7504 afr_write_subvol_set(call_frame_t *frame, xlator_t *this)
7505 {
7506 afr_local_t *local = NULL;
7507 afr_private_t *priv = NULL;
7508 unsigned char *data_accused = NULL;
7509 unsigned char *metadata_accused = NULL;
7510 unsigned char *data_readable = NULL;
7511 unsigned char *metadata_readable = NULL;
7512 uint16_t datamap = 0;
7513 uint16_t metadatamap = 0;
7514 uint64_t val = 0;
7515 int event = 0;
7516 int i = 0;
7517
7518 local = frame->local;
7519 priv = this->private;
7520 data_accused = alloca0(priv->child_count);
7521 metadata_accused = alloca0(priv->child_count);
7522 data_readable = alloca0(priv->child_count);
7523 metadata_readable = alloca0(priv->child_count);
7524 event = local->event_generation;
7525
7526 afr_readables_fill(local, this, local->inode, data_accused,
7527 metadata_accused, data_readable, metadata_readable,
7528 NULL);
7529
7530 for (i = 0; i < priv->child_count; i++) {
7531 if (data_readable[i])
7532 datamap |= (1 << i);
7533 if (metadata_readable[i])
7534 metadatamap |= (1 << i);
7535 }
7536
7537 val = ((uint64_t)metadatamap) | (((uint64_t)datamap) << 16) |
7538 (((uint64_t)event) << 32);
7539
7540 LOCK(&local->inode->lock);
7541 {
7542 if (local->inode_ctx->write_subvol == 0 &&
7543 local->transaction.type == AFR_DATA_TRANSACTION) {
7544 local->inode_ctx->write_subvol = val;
7545 }
7546 }
7547 UNLOCK(&local->inode->lock);
7548
7549 return 0;
7550 }
7551
7552 int
7553 afr_write_subvol_reset(call_frame_t *frame, xlator_t *this)
7554 {
7555 afr_local_t *local = NULL;
7556
7557 local = frame->local;
7558 LOCK(&local->inode->lock);
7559 {
7560 GF_ASSERT(local->inode_ctx->lock_count > 0);
7561 local->inode_ctx->lock_count--;
7562
7563 if (!local->inode_ctx->lock_count)
7564 local->inode_ctx->write_subvol = 0;
7565 }
7566 UNLOCK(&local->inode->lock);
7567
7568 return 0;
7569 }
7570
7571 int
7572 afr_set_inode_local(xlator_t *this, afr_local_t *local, inode_t *inode)
7573 {
7574 afr_inode_ctx_t *inode_ctx = NULL;
7575
7576 local->inode = inode_ref(inode);
7577 LOCK(&local->inode->lock);
7578 {
7579 inode_ctx = __afr_inode_ctx_get(this, local->inode);
7580 if (inode_ctx) {
7581 local->inode_ctx = inode_ctx;
7582 }
7583 }
7584 UNLOCK(&local->inode->lock);
7585 if (inode_ctx == NULL) {
7586 gf_msg_callingfn(
7587 this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_INODE_CTX_GET_FAILED,
7588 "Error getting inode ctx %s", uuid_utoa(local->inode->gfid));
7589 return -1;
7590 }
7591
7592 return 0;
7593 }
7594
7595 gf_boolean_t
7596 afr_ta_is_fop_called_from_synctask(xlator_t *this)
7597 {
7598 struct synctask *task = NULL;
7599 gf_lkowner_t tmp_owner;
7600
7601 task = synctask_get();
7602 if (!task)
7603 return _gf_false;
7604
7605 set_lk_owner_from_ptr(&tmp_owner, (void *)this);
7606
7607 if (!is_same_lkowner(&tmp_owner, &task->frame->root->lk_owner))
7608 return _gf_false;
7609
7610 return _gf_true;
7611 }
7612
7613 int
7614 afr_ta_post_op_lock(xlator_t *this, loc_t *loc)
7615 {
7616 int ret = 0;
7617 uuid_t gfid = {
7618 0,
7619 };
7620 afr_private_t *priv = this->private;
7621 gf_boolean_t locked = _gf_false;
7622 struct gf_flock flock1 = {
7623 0,
7624 };
7625 struct gf_flock flock2 = {
7626 0,
7627 };
7628 int32_t cmd = 0;
7629
7630 /* Clients must take AFR_TA_DOM_NOTIFY lock only when the previous lock
7631 * has been released in afr_notify due to upcall notification from shd.
7632 */
7633 GF_ASSERT(priv->ta_notify_dom_lock_offset == 0);
7634
7635 if (!priv->shd.iamshd)
7636 GF_ASSERT(afr_ta_is_fop_called_from_synctask(this));
7637 flock1.l_type = F_WRLCK;
7638
7639 while (!locked) {
7640 if (priv->shd.iamshd) {
7641 cmd = F_SETLKW;
7642 flock1.l_start = 0;
7643 flock1.l_len = 0;
7644 } else {
7645 cmd = F_SETLK;
7646 gf_uuid_generate(gfid);
7647 flock1.l_start = gfid_to_ino(gfid);
7648 if (flock1.l_start < 0)
7649 flock1.l_start = -flock1.l_start;
7650 flock1.l_len = 1;
7651 }
7652 ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
7653 AFR_TA_DOM_NOTIFY, loc, cmd, &flock1, NULL, NULL);
7654 if (!ret) {
7655 locked = _gf_true;
7656 priv->ta_notify_dom_lock_offset = flock1.l_start;
7657 } else if (ret == -EAGAIN) {
7658 continue;
7659 } else {
7660 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
7661 "Failed to get "
7662 "AFR_TA_DOM_NOTIFY lock on %s.",
7663 loc->name);
7664 goto out;
7665 }
7666 }
7667
7668 flock2.l_type = F_WRLCK;
7669 flock2.l_start = 0;
7670 flock2.l_len = 0;
7671 ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
7672 AFR_TA_DOM_MODIFY, loc, F_SETLKW, &flock2, NULL, NULL);
7673 if (ret) {
7674 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
7675 "Failed to get AFR_TA_DOM_MODIFY lock on %s.", loc->name);
7676 flock1.l_type = F_UNLCK;
7677 ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
7678 AFR_TA_DOM_NOTIFY, loc, F_SETLK, &flock1, NULL,
7679 NULL);
7680 }
7681 out:
7682 return ret;
7683 }
7684
7685 int
7686 afr_ta_post_op_unlock(xlator_t *this, loc_t *loc)
7687 {
7688 afr_private_t *priv = this->private;
7689 struct gf_flock flock = {
7690 0,
7691 };
7692 int ret = 0;
7693
7694 if (!priv->shd.iamshd)
7695 GF_ASSERT(afr_ta_is_fop_called_from_synctask(this));
7696 flock.l_type = F_UNLCK;
7697 flock.l_start = 0;
7698 flock.l_len = 0;
7699
7700 ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
7701 AFR_TA_DOM_MODIFY, loc, F_SETLK, &flock, NULL, NULL);
7702 if (ret) {
7703 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
7704 "Failed to unlock AFR_TA_DOM_MODIFY lock.");
7705 goto out;
7706 }
7707
7708 if (!priv->shd.iamshd)
7709 /* Mounts (clients) will not release the AFR_TA_DOM_NOTIFY lock
7710 * in post-op as they use it as a notification mechanism. When
7711 * shd sends a lock request on TA during heal, the clients will
7712 * receive a lock-contention upcall notification upon which they
7713 * will release the AFR_TA_DOM_NOTIFY lock after completing the
7714 * in flight I/O.*/
7715 goto out;
7716
7717 ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
7718 AFR_TA_DOM_NOTIFY, loc, F_SETLK, &flock, NULL, NULL);
7719 if (ret) {
7720 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
7721 "Failed to unlock AFR_TA_DOM_NOTIFY lock.");
7722 }
7723 out:
7724 return ret;
7725 }
7726
7727 call_frame_t *
7728 afr_ta_frame_create(xlator_t *this)
7729 {
7730 call_frame_t *frame = NULL;
7731 void *lk_owner = NULL;
7732
7733 frame = create_frame(this, this->ctx->pool);
7734 if (!frame)
7735 return NULL;
7736 lk_owner = (void *)this;
7737 afr_set_lk_owner(frame, this, lk_owner);
7738 return frame;
7739 }
7740
7741 gf_boolean_t
7742 afr_ta_has_quorum(afr_private_t *priv, afr_local_t *local)
7743 {
7744 int data_count = 0;
7745
7746 data_count = AFR_COUNT(local->child_up, priv->child_count);
7747 if (data_count == 2) {
7748 return _gf_true;
7749 } else if (data_count == 1 && local->ta_child_up) {
7750 return _gf_true;
7751 }
7752
7753 return _gf_false;
7754 }
7755
7756 static gf_boolean_t
7757 afr_is_add_replica_mount_lookup_on_root(call_frame_t *frame)
7758 {
7759 afr_local_t *local = NULL;
7760
7761 if (frame->root->pid != GF_CLIENT_PID_ADD_REPLICA_MOUNT)
7762 return _gf_false;
7763
7764 local = frame->local;
7765
7766 if (local->op != GF_FOP_LOOKUP)
7767 /* TODO:If the replica count is being increased on a plain distribute
7768 * volume that was never mounted, we need to allow setxattr on '/' with
7769 * GF_CLIENT_PID_NO_ROOT_SQUASH to accomodate for DHT layout setting */
7770 return _gf_false;
7771
7772 if (local->inode == NULL)
7773 return _gf_false;
7774
7775 if (!__is_root_gfid(local->inode->gfid))
7776 return _gf_false;
7777
7778 return _gf_true;
7779 }
7780
7781 gf_boolean_t
7782 afr_lookup_has_quorum(call_frame_t *frame, const unsigned int up_children_count)
7783 {
7784 if (frame && (up_children_count > 0) &&
7785 afr_is_add_replica_mount_lookup_on_root(frame))
7786 return _gf_true;
7787
7788 return _gf_false;
7789 }
7790
7791 void
7792 afr_handle_replies_quorum(call_frame_t *frame, xlator_t *this)
7793 {
7794 afr_local_t *local = frame->local;
7795 afr_private_t *priv = this->private;
7796 unsigned char *success_replies = NULL;
7797
7798 success_replies = alloca0(priv->child_count);
7799 afr_fill_success_replies(local, priv, success_replies);
7800
7801 if (priv->quorum_count && !afr_has_quorum(success_replies, priv, NULL)) {
7802 local->op_errno = afr_final_errno(local, priv);
7803 if (!local->op_errno)
7804 local->op_errno = afr_quorum_errno(priv);
7805 local->op_ret = -1;
7806 }
7807 }
7808
7809 gf_boolean_t
7810 afr_ta_dict_contains_pending_xattr(dict_t *dict, afr_private_t *priv, int child)
7811 {
7812 int *pending = NULL;
7813 int ret = 0;
7814 int i = 0;
7815
7816 ret = dict_get_ptr(dict, priv->pending_key[child], (void *)&pending);
7817 if (ret == 0) {
7818 for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++) {
7819 if (pending[i]) {
7820 return _gf_true;
7821 }
7822 }
7823 }
7824
7825 return _gf_false;
7826 }