Debian Codegraph @

xlators/cluster/afr/src/afr-transaction.c

raw
1 /*
2 Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
3 This file is part of GlusterFS.
4
5 This file is licensed to you under your choice of the GNU Lesser
6 General Public License, version 3 or any later version (LGPLv3 or
7 later), or the GNU General Public License, version 2 (GPLv2), in all
8 cases as published by the Free Software Foundation.
9 */
10
11 #include <glusterfs/dict.h>
12 #include <glusterfs/timer.h>
13
14 #include "afr-transaction.h"
15 #include "afr-self-heal.h"
16 #include "afr-messages.h"
17
18 #include <signal.h>
19
20 typedef enum {
21 AFR_TRANSACTION_PRE_OP,
22 AFR_TRANSACTION_POST_OP,
23 } afr_xattrop_type_t;
24
25 static void
26 afr_lock_resume_shared(struct list_head *list);
27
28 static void
29 afr_post_op_handle_success(call_frame_t *frame, xlator_t *this);
30
31 static void
32 afr_post_op_handle_failure(call_frame_t *frame, xlator_t *this, int op_errno);
33
34 static int
35 afr_internal_lock_finish(call_frame_t *frame, xlator_t *this);
36
37 static void
38 __afr_transaction_wake_shared(afr_local_t *local, struct list_head *shared);
39
40 static void
41 afr_changelog_post_op_do(call_frame_t *frame, xlator_t *this);
42
43 static int
44 afr_changelog_post_op_safe(call_frame_t *frame, xlator_t *this);
45
46 static gf_boolean_t
47 afr_changelog_pre_op_uninherit(call_frame_t *frame, xlator_t *this);
48
49 static gf_boolean_t
50 afr_changelog_pre_op_update(call_frame_t *frame, xlator_t *this);
51
52 static int
53 afr_changelog_call_count(afr_transaction_type type,
54 unsigned char *pre_op_subvols,
55 unsigned char *failed_subvols,
56 unsigned int child_count);
57 static int
58 afr_changelog_do(call_frame_t *frame, xlator_t *this, dict_t *xattr,
59 afr_changelog_resume_t changelog_resume,
60 afr_xattrop_type_t op);
61
62 static void
63 afr_ta_decide_post_op_state(call_frame_t *frame, xlator_t *this);
64
65 static int
66 afr_ta_post_op_do(void *opaque);
67
68 static int
69 afr_ta_post_op_synctask(xlator_t *this, afr_local_t *local);
70
71 static int
72 afr_changelog_post_op_done(call_frame_t *frame, xlator_t *this);
73
74 static void
75 afr_changelog_post_op_fail(call_frame_t *frame, xlator_t *this, int op_errno);
76
77 void
78 afr_ta_locked_priv_invalidate(afr_private_t *priv)
79 {
80 priv->ta_bad_child_index = AFR_CHILD_UNKNOWN;
81 priv->release_ta_notify_dom_lock = _gf_false;
82 priv->ta_notify_dom_lock_offset = 0;
83 }
84
85 static void
86 afr_ta_process_waitq(xlator_t *this)
87 {
88 afr_local_t *entry = NULL;
89 afr_private_t *priv = this->private;
90 struct list_head waitq = {
91 0,
92 };
93
94 INIT_LIST_HEAD(&waitq);
95 LOCK(&priv->lock);
96 list_splice_init(&priv->ta_waitq, &waitq);
97 UNLOCK(&priv->lock);
98 list_for_each_entry(entry, &waitq, ta_waitq)
99 {
100 afr_ta_decide_post_op_state(entry->transaction.frame, this);
101 }
102 }
103
104 int
105 afr_ta_lock_release_done(int ret, call_frame_t *ta_frame, void *opaque)
106 {
107 afr_ta_process_waitq(ta_frame->this);
108 STACK_DESTROY(ta_frame->root);
109 return 0;
110 }
111
112 int
113 afr_release_notify_lock_for_ta(void *opaque)
114 {
115 xlator_t *this = NULL;
116 afr_private_t *priv = NULL;
117 loc_t loc = {
118 0,
119 };
120 struct gf_flock flock = {
121 0,
122 };
123 int ret = -1;
124
125 this = (xlator_t *)opaque;
126 priv = this->private;
127 ret = afr_fill_ta_loc(this, &loc, _gf_true);
128 if (ret) {
129 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
130 "Failed to populate loc for thin-arbiter.");
131 goto out;
132 }
133 flock.l_type = F_UNLCK;
134 flock.l_start = priv->ta_notify_dom_lock_offset;
135 flock.l_len = 1;
136 ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
137 AFR_TA_DOM_NOTIFY, &loc, F_SETLK, &flock, NULL, NULL);
138 if (ret) {
139 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
140 "Failed to unlock AFR_TA_DOM_NOTIFY lock.");
141 }
142
143 LOCK(&priv->lock);
144 {
145 afr_ta_locked_priv_invalidate(priv);
146 }
147 UNLOCK(&priv->lock);
148 out:
149 loc_wipe(&loc);
150 return ret;
151 }
152
153 static void
154 gf_zero_fill_stat(struct iatt *buf)
155 {
156 buf->ia_nlink = 0;
157 buf->ia_ctime = 0;
158 }
159
160 void
161 afr_zero_fill_stat(afr_local_t *local)
162 {
163 if (!local)
164 return;
165 if (local->transaction.type == AFR_DATA_TRANSACTION ||
166 local->transaction.type == AFR_METADATA_TRANSACTION) {
167 gf_zero_fill_stat(&local->cont.inode_wfop.prebuf);
168 gf_zero_fill_stat(&local->cont.inode_wfop.postbuf);
169 } else if (local->transaction.type == AFR_ENTRY_TRANSACTION ||
170 local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
171 gf_zero_fill_stat(&local->cont.dir_fop.buf);
172 gf_zero_fill_stat(&local->cont.dir_fop.preparent);
173 gf_zero_fill_stat(&local->cont.dir_fop.postparent);
174 if (local->transaction.type == AFR_ENTRY_TRANSACTION)
175 return;
176 gf_zero_fill_stat(&local->cont.dir_fop.prenewparent);
177 gf_zero_fill_stat(&local->cont.dir_fop.postnewparent);
178 }
179 }
180
181 /* In case of errors afr needs to choose which xdata from lower xlators it needs
182 * to unwind with. The way it is done is by checking if there are
183 * any good subvols which failed. Give preference to errnos other than
184 * ENOTCONN even if the child is source */
185 void
186 afr_pick_error_xdata(afr_local_t *local, unsigned int child_count,
187 inode_t *inode1, unsigned char *readable1, inode_t *inode2,
188 unsigned char *readable2)
189 {
190 int s = -1; /*selection*/
191 int i = 0;
192 unsigned char *readable = NULL;
193
194 if (local->xdata_rsp) {
195 dict_unref(local->xdata_rsp);
196 local->xdata_rsp = NULL;
197 }
198
199 readable = alloca0(child_count * sizeof(*readable));
200 if (inode2 && readable2) { /*rename fop*/
201 AFR_INTERSECT(readable, readable1, readable2, child_count);
202 } else {
203 memcpy(readable, readable1, sizeof(*readable) * child_count);
204 }
205
206 for (i = 0; i < child_count; i++) {
207 if (!local->replies[i].valid)
208 continue;
209
210 if (local->replies[i].op_ret >= 0)
211 continue;
212
213 if (local->replies[i].op_errno == ENOTCONN)
214 continue;
215
216 /*Order is important in the following condition*/
217 if ((s < 0) || (!readable[s] && readable[i]))
218 s = i;
219 }
220
221 if (s != -1 && local->replies[s].xdata) {
222 local->xdata_rsp = dict_ref(local->replies[s].xdata);
223 } else if (s == -1) {
224 for (i = 0; i < child_count; i++) {
225 if (!local->replies[i].valid)
226 continue;
227
228 if (local->replies[i].op_ret >= 0)
229 continue;
230
231 if (!local->replies[i].xdata)
232 continue;
233 local->xdata_rsp = dict_ref(local->replies[i].xdata);
234 break;
235 }
236 }
237 }
238
239 gf_boolean_t
240 afr_needs_changelog_update(afr_local_t *local)
241 {
242 if (local->transaction.type == AFR_DATA_TRANSACTION)
243 return _gf_true;
244 if (!local->optimistic_change_log)
245 return _gf_true;
246 return _gf_false;
247 }
248
249 static gf_boolean_t
250 afr_changelog_has_quorum(afr_local_t *local, xlator_t *this)
251 {
252 afr_private_t *priv = NULL;
253 int i = 0;
254 unsigned char *success_children = NULL;
255
256 priv = this->private;
257 success_children = alloca0(priv->child_count);
258
259 for (i = 0; i < priv->child_count; i++) {
260 if (!local->transaction.failed_subvols[i]) {
261 success_children[i] = 1;
262 }
263 }
264
265 if (afr_has_quorum(success_children, priv, NULL)) {
266 return _gf_true;
267 }
268
269 return _gf_false;
270 }
271
272 static gf_boolean_t
273 afr_is_write_subvol_valid(call_frame_t *frame, xlator_t *this)
274 {
275 int i = 0;
276 afr_local_t *local = NULL;
277 afr_private_t *priv = NULL;
278 uint64_t write_subvol = 0;
279 unsigned char *writable = NULL;
280 uint16_t datamap = 0;
281
282 local = frame->local;
283 priv = this->private;
284 writable = alloca0(priv->child_count);
285
286 write_subvol = afr_write_subvol_get(frame, this);
287 datamap = (write_subvol & 0x00000000ffff0000) >> 16;
288 for (i = 0; i < priv->child_count; i++) {
289 if (datamap & (1 << i))
290 writable[i] = 1;
291
292 if (writable[i] && !local->transaction.failed_subvols[i])
293 return _gf_true;
294 }
295
296 return _gf_false;
297 }
298
299 int
300 afr_transaction_fop(call_frame_t *frame, xlator_t *this)
301 {
302 afr_local_t *local = NULL;
303 afr_private_t *priv = NULL;
304 int call_count = -1;
305 unsigned char *failed_subvols = NULL;
306 int i = 0;
307
308 local = frame->local;
309 priv = this->private;
310
311 failed_subvols = local->transaction.failed_subvols;
312 call_count = priv->child_count -
313 AFR_COUNT(failed_subvols, priv->child_count);
314 /* Fail if pre-op did not succeed on quorum no. of bricks. */
315 if (!afr_changelog_has_quorum(local, this) || !call_count) {
316 local->op_ret = -1;
317 /* local->op_errno is already captured in changelog cbk. */
318 afr_transaction_resume(frame, this);
319 return 0;
320 }
321
322 /* Fail if at least one writeable brick isn't up.*/
323 if (local->transaction.type == AFR_DATA_TRANSACTION &&
324 !afr_is_write_subvol_valid(frame, this)) {
325 local->op_ret = -1;
326 local->op_errno = EIO;
327 afr_transaction_resume(frame, this);
328 return 0;
329 }
330
331 local->call_count = call_count;
332 for (i = 0; i < priv->child_count; i++) {
333 if (local->transaction.pre_op[i] && !failed_subvols[i]) {
334 local->transaction.wind(frame, this, i);
335
336 if (!--call_count)
337 break;
338 }
339 }
340
341 return 0;
342 }
343
344 static int
345 afr_transaction_done(call_frame_t *frame, xlator_t *this)
346 {
347 afr_local_t *local = NULL;
348 afr_private_t *priv = NULL;
349 gf_boolean_t unwind = _gf_false;
350 afr_lock_t *lock = NULL;
351 afr_local_t *lock_local = NULL;
352
353 priv = this->private;
354 local = frame->local;
355
356 if (priv->consistent_metadata) {
357 LOCK(&frame->lock);
358 {
359 unwind = (local->transaction.main_frame != NULL);
360 }
361 UNLOCK(&frame->lock);
362 if (unwind) /*It definitely did post-op*/
363 afr_zero_fill_stat(local);
364 }
365
366 if (local->transaction.do_eager_unlock) {
367 lock = &local->inode_ctx->lock[local->transaction.type];
368 LOCK(&local->inode->lock);
369 {
370 lock->acquired = _gf_false;
371 lock->release = _gf_false;
372 list_splice_init(&lock->frozen, &lock->waiting);
373 if (list_empty(&lock->waiting))
374 goto unlock;
375 lock_local = list_entry(lock->waiting.next, afr_local_t,
376 transaction.wait_list);
377 list_del_init(&lock_local->transaction.wait_list);
378 list_add(&lock_local->transaction.owner_list, &lock->owners);
379 }
380 unlock:
381 UNLOCK(&local->inode->lock);
382 }
383 if (lock_local) {
384 afr_lock(lock_local->transaction.frame,
385 lock_local->transaction.frame->this);
386 }
387 local->transaction.unwind(frame, this);
388
389 GF_ASSERT(list_empty(&local->transaction.owner_list));
390 GF_ASSERT(list_empty(&local->transaction.wait_list));
391 AFR_STACK_DESTROY(frame);
392
393 return 0;
394 }
395
396 static void
397 afr_lock_fail_shared(afr_local_t *local, struct list_head *list)
398 {
399 afr_local_t *each = NULL;
400
401 while (!list_empty(list)) {
402 each = list_entry(list->next, afr_local_t, transaction.wait_list);
403 list_del_init(&each->transaction.wait_list);
404 each->op_ret = -1;
405 each->op_errno = local->op_errno;
406 afr_transaction_done(each->transaction.frame,
407 each->transaction.frame->this);
408 }
409 }
410
411 static void
412 afr_handle_lock_acquire_failure(afr_local_t *local)
413 {
414 struct list_head shared;
415 afr_lock_t *lock = NULL;
416
417 if (!local->transaction.eager_lock_on)
418 goto out;
419
420 lock = &local->inode_ctx->lock[local->transaction.type];
421
422 INIT_LIST_HEAD(&shared);
423 LOCK(&local->inode->lock);
424 {
425 lock->release = _gf_true;
426 list_splice_init(&lock->waiting, &shared);
427 }
428 UNLOCK(&local->inode->lock);
429
430 afr_lock_fail_shared(local, &shared);
431 local->transaction.do_eager_unlock = _gf_true;
432 out:
433 local->internal_lock.lock_cbk = afr_transaction_done;
434 afr_unlock(local->transaction.frame, local->transaction.frame->this);
435 }
436
437 call_frame_t *
438 afr_transaction_detach_fop_frame(call_frame_t *frame)
439 {
440 afr_local_t *local = NULL;
441 call_frame_t *fop_frame = NULL;
442
443 local = frame->local;
444
445 afr_handle_inconsistent_fop(frame, &local->op_ret, &local->op_errno);
446 LOCK(&frame->lock);
447 {
448 fop_frame = local->transaction.main_frame;
449 local->transaction.main_frame = NULL;
450 }
451 UNLOCK(&frame->lock);
452
453 return fop_frame;
454 }
455
456 static void
457 afr_save_lk_owner(call_frame_t *frame)
458 {
459 afr_local_t *local = NULL;
460
461 local = frame->local;
462
463 lk_owner_copy(&local->saved_lk_owner, &frame->root->lk_owner);
464 }
465
466 static void
467 afr_restore_lk_owner(call_frame_t *frame)
468 {
469 afr_local_t *local = NULL;
470
471 local = frame->local;
472
473 lk_owner_copy(&frame->root->lk_owner, &local->saved_lk_owner);
474 }
475
476 void
477 __mark_all_success(call_frame_t *frame, xlator_t *this)
478 {
479 afr_private_t *priv = NULL;
480 afr_local_t *local = NULL;
481 int i;
482
483 local = frame->local;
484 priv = this->private;
485
486 for (i = 0; i < priv->child_count; i++) {
487 local->transaction.failed_subvols[i] = 0;
488 }
489 }
490
491 static void
492 afr_compute_pre_op_sources(call_frame_t *frame, xlator_t *this)
493 {
494 afr_local_t *local = NULL;
495 afr_private_t *priv = NULL;
496 afr_transaction_type type = -1;
497 dict_t *xdata = NULL;
498 int **matrix = NULL;
499 int idx = -1;
500 int i = 0;
501 int j = 0;
502
503 priv = this->private;
504 local = frame->local;
505 type = local->transaction.type;
506 idx = afr_index_for_transaction_type(type);
507 matrix = ALLOC_MATRIX(priv->child_count, int);
508
509 for (i = 0; i < priv->child_count; i++) {
510 if (!local->transaction.changelog_xdata[i])
511 continue;
512 xdata = local->transaction.changelog_xdata[i];
513 afr_selfheal_fill_matrix(this, matrix, i, idx, xdata);
514 }
515
516 memset(local->transaction.pre_op_sources, 1, priv->child_count);
517
518 /*If lock or pre-op failed on a brick, it is not a source. */
519 for (i = 0; i < priv->child_count; i++) {
520 if (local->transaction.failed_subvols[i])
521 local->transaction.pre_op_sources[i] = 0;
522 }
523
524 /* If brick is blamed by others, it is not a source. */
525 for (i = 0; i < priv->child_count; i++)
526 for (j = 0; j < priv->child_count; j++)
527 if (matrix[i][j] != 0)
528 local->transaction.pre_op_sources[j] = 0;
529 }
530
531 static void
532 afr_txn_arbitrate_fop(call_frame_t *frame, xlator_t *this)
533 {
534 afr_local_t *local = NULL;
535 afr_private_t *priv = NULL;
536 int pre_op_sources_count = 0;
537 int i = 0;
538
539 priv = this->private;
540 local = frame->local;
541
542 afr_compute_pre_op_sources(frame, this);
543 pre_op_sources_count = AFR_COUNT(local->transaction.pre_op_sources,
544 priv->child_count);
545
546 /* If arbiter is the only source, do not proceed. */
547 if (pre_op_sources_count < 2 &&
548 local->transaction.pre_op_sources[ARBITER_BRICK_INDEX]) {
549 local->op_ret = -1;
550 local->op_errno = ENOTCONN;
551 for (i = 0; i < priv->child_count; i++)
552 local->transaction.failed_subvols[i] = 1;
553 }
554
555 afr_transaction_fop(frame, this);
556
557 return;
558 }
559
560 static int
561 afr_transaction_perform_fop(call_frame_t *frame, xlator_t *this)
562 {
563 afr_local_t *local = NULL;
564 afr_private_t *priv = NULL;
565 int i = 0;
566 int ret = 0;
567 int failure_count = 0;
568 struct list_head shared;
569 afr_lock_t *lock = NULL;
570
571 local = frame->local;
572 priv = this->private;
573
574 INIT_LIST_HEAD(&shared);
575 if (local->transaction.type == AFR_DATA_TRANSACTION &&
576 !local->transaction.inherited) {
577 ret = afr_write_subvol_set(frame, this);
578 if (ret) {
579 /*act as if operation failed on all subvols*/
580 local->op_ret = -1;
581 local->op_errno = -ret;
582 for (i = 0; i < priv->child_count; i++)
583 local->transaction.failed_subvols[i] = 1;
584 }
585 }
586
587 if (local->pre_op_compat)
588 /* old mode, pre-op was done as afr_changelog_do()
589 just now, before OP */
590 afr_changelog_pre_op_update(frame, this);
591
592 if (!local->transaction.eager_lock_on || local->transaction.inherited)
593 goto fop;
594 failure_count = AFR_COUNT(local->transaction.failed_subvols,
595 priv->child_count);
596 if (failure_count == priv->child_count) {
597 afr_handle_lock_acquire_failure(local);
598 return 0;
599 } else {
600 lock = &local->inode_ctx->lock[local->transaction.type];
601 LOCK(&local->inode->lock);
602 {
603 lock->acquired = _gf_true;
604 __afr_transaction_wake_shared(local, &shared);
605 }
606 UNLOCK(&local->inode->lock);
607 }
608
609 fop:
610 /* Perform fops with the lk-owner from top xlator.
611 * Eg: lk-owner of posix-lk and flush should be same,
612 * flush cant clear the posix-lks without that lk-owner.
613 */
614 afr_save_lk_owner(frame);
615 lk_owner_copy(&frame->root->lk_owner,
616 &local->transaction.main_frame->root->lk_owner);
617
618 if (priv->arbiter_count == 1) {
619 afr_txn_arbitrate_fop(frame, this);
620 } else {
621 afr_transaction_fop(frame, this);
622 }
623
624 afr_lock_resume_shared(&shared);
625 return 0;
626 }
627
628 int
629 afr_set_pending_dict(afr_private_t *priv, dict_t *xattr, int **pending)
630 {
631 int i = 0;
632 int ret = 0;
633
634 for (i = 0; i < priv->child_count; i++) {
635 ret = dict_set_static_bin(xattr, priv->pending_key[i], pending[i],
636 AFR_NUM_CHANGE_LOGS * sizeof(int));
637 /* 3 = data+metadata+entry */
638
639 if (ret)
640 break;
641 }
642
643 return ret;
644 }
645
646 static void
647 afr_ta_dom_lock_check_and_release(afr_ta_fop_state_t fop_state, xlator_t *this)
648 {
649 afr_private_t *priv = this->private;
650 unsigned int inmem_count = 0;
651 unsigned int onwire_count = 0;
652 gf_boolean_t release = _gf_false;
653
654 LOCK(&priv->lock);
655 {
656 /*Once we get notify lock release upcall notification,
657 if any of the fop state counters are non-zero, we will
658 not release the lock.
659 */
660 onwire_count = priv->ta_on_wire_txn_count;
661 inmem_count = priv->ta_in_mem_txn_count;
662 switch (fop_state) {
663 case TA_GET_INFO_FROM_TA_FILE:
664 onwire_count = --priv->ta_on_wire_txn_count;
665 break;
666 case TA_INFO_IN_MEMORY_SUCCESS:
667 case TA_INFO_IN_MEMORY_FAILED:
668 inmem_count = --priv->ta_in_mem_txn_count;
669 break;
670 case TA_WAIT_FOR_NOTIFY_LOCK_REL:
671 GF_ASSERT(0);
672 break;
673 case TA_SUCCESS:
674 break;
675 }
676 release = priv->release_ta_notify_dom_lock;
677 }
678 UNLOCK(&priv->lock);
679
680 if (inmem_count != 0 || release == _gf_false || onwire_count != 0)
681 return;
682
683 afr_ta_lock_release_synctask(this);
684 }
685
686 static void
687 afr_ta_process_onwireq(afr_ta_fop_state_t fop_state, xlator_t *this)
688 {
689 afr_private_t *priv = this->private;
690 afr_local_t *entry = NULL;
691 int bad_child = AFR_CHILD_UNKNOWN;
692
693 struct list_head onwireq = {
694 0,
695 };
696 INIT_LIST_HEAD(&onwireq);
697
698 LOCK(&priv->lock);
699 {
700 bad_child = priv->ta_bad_child_index;
701 if (bad_child == AFR_CHILD_UNKNOWN) {
702 /*The previous on-wire ta_post_op was a failure. Just dequeue
703 *one element to wind on-wire again. */
704 entry = list_entry(priv->ta_onwireq.next, afr_local_t, ta_onwireq);
705 list_del_init(&entry->ta_onwireq);
706 } else {
707 /* Prepare to process all fops based on bad_child_index. */
708 list_splice_init(&priv->ta_onwireq, &onwireq);
709 }
710 }
711 UNLOCK(&priv->lock);
712
713 if (entry) {
714 afr_ta_post_op_synctask(this, entry);
715 return;
716 } else {
717 while (!list_empty(&onwireq)) {
718 entry = list_entry(onwireq.next, afr_local_t, ta_onwireq);
719 list_del_init(&entry->ta_onwireq);
720 if (entry->ta_failed_subvol == bad_child) {
721 afr_post_op_handle_success(entry->transaction.frame, this);
722 } else {
723 afr_post_op_handle_failure(entry->transaction.frame, this, EIO);
724 }
725 }
726 }
727 }
728
729 static int
730 afr_changelog_post_op_done(call_frame_t *frame, xlator_t *this)
731 {
732 afr_local_t *local = NULL;
733 afr_internal_lock_t *int_lock = NULL;
734 afr_private_t *priv = NULL;
735
736 local = frame->local;
737 priv = this->private;
738 int_lock = &local->internal_lock;
739
740 if (priv->thin_arbiter_count) {
741 /*fop should not come here with TA_WAIT_FOR_NOTIFY_LOCK_REL state */
742 afr_ta_dom_lock_check_and_release(local->fop_state, this);
743 }
744
745 /* Fail the FOP if post-op did not succeed on quorum no. of bricks. */
746 if (!afr_changelog_has_quorum(local, this)) {
747 local->op_ret = -1;
748 /*local->op_errno is already captured in changelog cbk*/
749 }
750
751 if (local->transaction.resume_stub) {
752 call_resume(local->transaction.resume_stub);
753 local->transaction.resume_stub = NULL;
754 }
755
756 int_lock->lock_cbk = afr_transaction_done;
757 afr_unlock(frame, this);
758
759 return 0;
760 }
761
762 static void
763 afr_changelog_post_op_fail(call_frame_t *frame, xlator_t *this, int op_errno)
764 {
765 afr_local_t *local = frame->local;
766 local->op_ret = -1;
767 local->op_errno = op_errno;
768
769 gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_THIN_ARB,
770 "Failing %s for gfid %s. Fop state is:%d", gf_fop_list[local->op],
771 uuid_utoa(local->inode->gfid), local->fop_state);
772
773 afr_changelog_post_op_done(frame, this);
774 }
775
776 unsigned char *
777 afr_locked_nodes_get(afr_transaction_type type, afr_internal_lock_t *int_lock)
778 {
779 /*Because same set of subvols participate in all lockee
780 * entities*/
781 return int_lock->lockee[0].locked_nodes;
782 }
783
784 static int
785 afr_changelog_call_count(afr_transaction_type type,
786 unsigned char *pre_op_subvols,
787 unsigned char *failed_subvols,
788 unsigned int child_count)
789 {
790 int i = 0;
791 int call_count = 0;
792
793 for (i = 0; i < child_count; i++) {
794 if (pre_op_subvols[i] && !failed_subvols[i]) {
795 call_count++;
796 }
797 }
798
799 if (type == AFR_ENTRY_RENAME_TRANSACTION)
800 call_count *= 2;
801
802 return call_count;
803 }
804
805 gf_boolean_t
806 afr_txn_nothing_failed(call_frame_t *frame, xlator_t *this)
807 {
808 afr_private_t *priv = NULL;
809 afr_local_t *local = NULL;
810 int i = 0;
811
812 local = frame->local;
813 priv = this->private;
814
815 if (priv->thin_arbiter_count) {
816 /* We need to perform post-op even if 1 data brick was down
817 * before the txn started.*/
818 if (AFR_COUNT(local->transaction.failed_subvols, priv->child_count))
819 return _gf_false;
820 }
821
822 for (i = 0; i < priv->child_count; i++) {
823 if (local->transaction.pre_op[i] &&
824 local->transaction.failed_subvols[i])
825 return _gf_false;
826 }
827
828 return _gf_true;
829 }
830
831 static void
832 afr_handle_symmetric_errors(call_frame_t *frame, xlator_t *this)
833 {
834 if (afr_is_symmetric_error(frame, this))
835 __mark_all_success(frame, this);
836 }
837
838 gf_boolean_t
839 afr_has_quorum(unsigned char *subvols, afr_private_t *priv, call_frame_t *frame)
840 {
841 unsigned int quorum_count = 0;
842 unsigned int up_children_count = 0;
843
844 up_children_count = AFR_COUNT(subvols, priv->child_count);
845
846 if (afr_lookup_has_quorum(frame, up_children_count))
847 return _gf_true;
848
849 if (priv->quorum_count == AFR_QUORUM_AUTO) {
850 /*
851 * Special case for auto-quorum with an even number of nodes.
852 *
853 * A replica set with even count N can only handle the same
854 * number of failures as odd N-1 before losing "vanilla"
855 * quorum, and the probability of more simultaneous failures is
856 * actually higher. For example, with a 1% chance of failure
857 * we'd have a 0.03% chance of two simultaneous failures with
858 * N=3 but a 0.06% chance with N=4. However, the special case
859 * is necessary for N=2 because there's no real quorum in that
860 * case (i.e. can't normally survive *any* failures). In that
861 * case, we treat the first node as a tie-breaker, allowing
862 * quorum to be retained in some cases while still honoring the
863 * all-important constraint that there can not simultaneously
864 * be two partitioned sets of nodes each believing they have
865 * quorum. Of two equally sized sets, the one without that
866 * first node will lose.
867 *
868 * It turns out that the special case is beneficial for higher
869 * values of N as well. Continuing the example above, the
870 * probability of losing quorum with N=4 and this type of
871 * quorum is (very) slightly lower than with N=3 and vanilla
872 * quorum. The difference becomes even more pronounced with
873 * higher N. Therefore, even though such replica counts are
874 * unlikely to be seen in practice, we might as well use the
875 * "special" quorum then as well.
876 */
877 if ((up_children_count * 2) == priv->child_count) {
878 return subvols[0];
879 }
880 }
881
882 if (priv->quorum_count == AFR_QUORUM_AUTO) {
883 quorum_count = priv->child_count / 2 + 1;
884 } else {
885 quorum_count = priv->quorum_count;
886 }
887
888 if (up_children_count >= quorum_count)
889 return _gf_true;
890
891 return _gf_false;
892 }
893
894 static gf_boolean_t
895 afr_has_fop_quorum(call_frame_t *frame)
896 {
897 xlator_t *this = frame->this;
898 afr_local_t *local = frame->local;
899 unsigned char *locked_nodes = NULL;
900
901 locked_nodes = afr_locked_nodes_get(local->transaction.type,
902 &local->internal_lock);
903 return afr_has_quorum(locked_nodes, this->private, NULL);
904 }
905
906 static gf_boolean_t
907 afr_has_fop_cbk_quorum(call_frame_t *frame)
908 {
909 afr_local_t *local = frame->local;
910 xlator_t *this = frame->this;
911 afr_private_t *priv = this->private;
912 unsigned char *success = alloca0(priv->child_count);
913 int i = 0;
914
915 for (i = 0; i < priv->child_count; i++) {
916 if (local->transaction.pre_op[i])
917 if (!local->transaction.failed_subvols[i])
918 success[i] = 1;
919 }
920
921 return afr_has_quorum(success, priv, NULL);
922 }
923
924 static gf_boolean_t
925 afr_need_dirty_marking(call_frame_t *frame, xlator_t *this)
926 {
927 afr_private_t *priv = this->private;
928 afr_local_t *local = NULL;
929 gf_boolean_t need_dirty = _gf_false;
930
931 local = frame->local;
932
933 if (!priv->quorum_count || !local->optimistic_change_log)
934 return _gf_false;
935
936 if (local->transaction.type == AFR_DATA_TRANSACTION ||
937 local->transaction.type == AFR_METADATA_TRANSACTION)
938 return _gf_false;
939
940 if (AFR_COUNT(local->transaction.failed_subvols, priv->child_count) ==
941 priv->child_count)
942 return _gf_false;
943
944 if (!afr_has_fop_cbk_quorum(frame))
945 need_dirty = _gf_true;
946
947 return need_dirty;
948 }
949
950 static void
951 afr_handle_quorum(call_frame_t *frame, xlator_t *this)
952 {
953 afr_local_t *local = NULL;
954 afr_private_t *priv = NULL;
955 int i = 0;
956 const char *file = NULL;
957 uuid_t gfid = {0};
958
959 local = frame->local;
960 priv = frame->this->private;
961
962 if (priv->quorum_count == 0)
963 return;
964
965 /* If the fop already failed return right away to preserve errno */
966 if (local->op_ret == -1)
967 return;
968
969 /*
970 * Network split may happen just after the fops are unwound, so check
971 * if the fop succeeded in a way it still follows quorum. If it doesn't,
972 * mark the fop as failure, mark the changelogs so it reflects that
973 * failure.
974 *
975 * Scenario:
976 * There are 3 mounts on 3 machines(node1, node2, node3) all writing to
977 * single file. Network split happened in a way that node1 can't see
978 * node2, node3. Node2, node3 both of them can't see node1. Now at the
979 * time of sending write all the bricks are up. Just after write fop is
980 * wound on node1, network split happens. Node1 thinks write fop failed
981 * on node2, node3 so marks pending changelog for those 2 extended
982 * attributes on node1. Node2, node3 thinks writes failed on node1 so
983 * they mark pending changelog for node1. When the network is stable
984 * again the file already is in split-brain. These checks prevent
985 * marking pending changelog on other subvolumes if the fop doesn't
986 * succeed in a way it is still following quorum. So with this fix what
987 * is happening is, node1 will have all pending changelog(FOOL) because
988 * the write succeeded only on node1 but failed on node2, node3 so
989 * instead of marking pending changelogs on node2, node3 it just treats
990 * the fop as failure and goes into DIRTY state. Where as node2, node3
991 * say they are sources and have pending changelog to node1 so there is
992 * no split-brain with the fix. The problem is eliminated completely.
993 */
994
995 if (afr_has_fop_cbk_quorum(frame))
996 return;
997
998 if (afr_need_dirty_marking(frame, this))
999 goto set_response;
1000
1001 for (i = 0; i < priv->child_count; i++) {
1002 if (local->transaction.pre_op[i])
1003 afr_transaction_fop_failed(local, i);
1004 }
1005
1006 set_response:
1007 local->op_ret = -1;
1008 local->op_errno = afr_final_errno(local, priv);
1009 if (local->op_errno == 0)
1010 local->op_errno = afr_quorum_errno(priv);
1011
1012 if (local->fd) {
1013 gf_uuid_copy(gfid, local->fd->inode->gfid);
1014 file = uuid_utoa(gfid);
1015 } else {
1016 loc_path(&local->loc, local->loc.name);
1017 file = local->loc.path;
1018 }
1019
1020 gf_msg(frame->this->name, GF_LOG_WARNING, local->op_errno,
1021 AFR_MSG_QUORUM_FAIL, "%s: Failing %s as quorum is not met", file,
1022 gf_fop_list[local->op]);
1023
1024 switch (local->transaction.type) {
1025 case AFR_ENTRY_TRANSACTION:
1026 case AFR_ENTRY_RENAME_TRANSACTION:
1027 afr_pick_error_xdata(local, priv->child_count, local->parent,
1028 local->readable, local->parent2,
1029 local->readable2);
1030 break;
1031 default:
1032 afr_pick_error_xdata(local, priv->child_count, local->inode,
1033 local->readable, NULL, NULL);
1034 break;
1035 }
1036 }
1037
1038 int
1039 afr_fill_ta_loc(xlator_t *this, loc_t *loc, gf_boolean_t is_gfid_based_fop)
1040 {
1041 afr_private_t *priv = NULL;
1042
1043 priv = this->private;
1044 loc->parent = inode_ref(this->itable->root);
1045 gf_uuid_copy(loc->pargfid, loc->parent->gfid);
1046 loc->name = priv->pending_key[THIN_ARBITER_BRICK_INDEX];
1047 if (is_gfid_based_fop && gf_uuid_is_null(priv->ta_gfid)) {
1048 /* Except afr_ta_id_file_check() which is path based, all other gluster
1049 * FOPS need gfid.*/
1050 return -EINVAL;
1051 }
1052 gf_uuid_copy(loc->gfid, priv->ta_gfid);
1053 loc->inode = inode_new(loc->parent->table);
1054 if (!loc->inode) {
1055 loc_wipe(loc);
1056 return -ENOMEM;
1057 }
1058 return 0;
1059 }
1060
1061 static int
1062 afr_ta_post_op_done(int ret, call_frame_t *frame, void *opaque)
1063 {
1064 xlator_t *this = NULL;
1065 afr_local_t *local = NULL;
1066 call_frame_t *txn_frame = NULL;
1067 afr_ta_fop_state_t fop_state;
1068
1069 local = (afr_local_t *)opaque;
1070 fop_state = local->fop_state;
1071 txn_frame = local->transaction.frame;
1072 this = frame->this;
1073
1074 if (ret == 0) {
1075 /*Mark pending xattrs on the up data brick.*/
1076 afr_post_op_handle_success(txn_frame, this);
1077 } else {
1078 afr_post_op_handle_failure(txn_frame, this, -ret);
1079 }
1080
1081 STACK_DESTROY(frame->root);
1082 afr_ta_process_onwireq(fop_state, this);
1083
1084 return 0;
1085 }
1086
1087 static int **
1088 afr_set_changelog_xattr(afr_private_t *priv, unsigned char *pending,
1089 dict_t *xattr, afr_local_t *local)
1090 {
1091 int **changelog = NULL;
1092 int idx = 0;
1093 int ret = 0;
1094 int i;
1095 uint32_t hton32_1;
1096
1097 if (local->is_new_entry == _gf_true) {
1098 changelog = afr_mark_pending_changelog(priv, pending, xattr,
1099 local->cont.dir_fop.buf.ia_type);
1100 } else {
1101 idx = afr_index_for_transaction_type(local->transaction.type);
1102 changelog = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
1103 if (!changelog) {
1104 goto out;
1105 }
1106 hton32_1 = htobe32(1);
1107 for (i = 0; i < priv->child_count; i++) {
1108 if (local->transaction.failed_subvols[i])
1109 changelog[i][idx] = hton32_1;
1110 }
1111 ret = afr_set_pending_dict(priv, xattr, changelog);
1112 if (ret < 0) {
1113 afr_matrix_cleanup(changelog, priv->child_count);
1114 return NULL;
1115 }
1116 }
1117
1118 out:
1119 return changelog;
1120 }
1121
1122 static void
1123 afr_ta_locked_xattrop_validate(afr_private_t *priv, afr_local_t *local,
1124 gf_boolean_t *valid)
1125 {
1126 if (priv->ta_event_gen > local->ta_event_gen) {
1127 /* We can't trust the ta's response anymore.*/
1128 afr_ta_locked_priv_invalidate(priv);
1129 *valid = _gf_false;
1130 return;
1131 }
1132 return;
1133 }
1134
1135 static int
1136 afr_ta_post_op_do(void *opaque)
1137 {
1138 afr_local_t *local = NULL;
1139 afr_private_t *priv = NULL;
1140 xlator_t *this = NULL;
1141 dict_t *xattr = NULL;
1142 unsigned char *pending = NULL;
1143 int **changelog = NULL;
1144 int failed_subvol = -1;
1145 int success_subvol = -1;
1146 loc_t loc = {
1147 0,
1148 };
1149 int i = 0;
1150 int ret = 0;
1151 gf_boolean_t valid = _gf_true;
1152
1153 local = (afr_local_t *)opaque;
1154 this = local->transaction.frame->this;
1155 priv = this->private;
1156
1157 ret = afr_fill_ta_loc(this, &loc, _gf_true);
1158 if (ret) {
1159 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
1160 "Failed to populate loc for thin-arbiter.");
1161 goto out;
1162 }
1163
1164 xattr = dict_new();
1165 if (!xattr) {
1166 ret = -ENOMEM;
1167 goto out;
1168 }
1169
1170 pending = alloca0(priv->child_count);
1171
1172 for (i = 0; i < priv->child_count; i++) {
1173 if (local->transaction.failed_subvols[i]) {
1174 pending[i] = 1;
1175 failed_subvol = i;
1176 } else {
1177 success_subvol = i;
1178 }
1179 }
1180
1181 changelog = afr_set_changelog_xattr(priv, pending, xattr, local);
1182
1183 if (!changelog) {
1184 ret = -ENOMEM;
1185 goto out;
1186 }
1187
1188 ret = afr_ta_post_op_lock(this, &loc);
1189 if (ret)
1190 goto out;
1191
1192 ret = syncop_xattrop(priv->children[THIN_ARBITER_BRICK_INDEX], &loc,
1193 GF_XATTROP_ADD_ARRAY, xattr, NULL, NULL, NULL);
1194 if (ret) {
1195 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
1196 "Post-op on thin-arbiter id file %s failed for gfid %s.",
1197 priv->pending_key[THIN_ARBITER_BRICK_INDEX],
1198 uuid_utoa(local->inode->gfid));
1199 }
1200 LOCK(&priv->lock);
1201 {
1202 if (ret == 0) {
1203 priv->ta_bad_child_index = failed_subvol;
1204 } else if (ret == -EINVAL) {
1205 priv->ta_bad_child_index = success_subvol;
1206 ret = -EIO; /* TA failed the fop. Return EIO to application. */
1207 }
1208
1209 afr_ta_locked_xattrop_validate(priv, local, &valid);
1210 }
1211 UNLOCK(&priv->lock);
1212 if (valid == _gf_false) {
1213 gf_msg(this->name, GF_LOG_ERROR, EIO, AFR_MSG_THIN_ARB,
1214 "Post-op on thin-arbiter id file %s for gfid %s invalidated due "
1215 "to event-gen mismatch.",
1216 priv->pending_key[THIN_ARBITER_BRICK_INDEX],
1217 uuid_utoa(local->inode->gfid));
1218 ret = -EIO;
1219 }
1220
1221 afr_ta_post_op_unlock(this, &loc);
1222 out:
1223 if (xattr)
1224 dict_unref(xattr);
1225
1226 if (changelog)
1227 afr_matrix_cleanup(changelog, priv->child_count);
1228
1229 loc_wipe(&loc);
1230
1231 return ret;
1232 }
1233
1234 static int
1235 afr_ta_post_op_synctask(xlator_t *this, afr_local_t *local)
1236 {
1237 call_frame_t *ta_frame = NULL;
1238 int ret = 0;
1239
1240 ta_frame = afr_ta_frame_create(this);
1241 if (!ta_frame) {
1242 gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
1243 "Failed to create ta_frame");
1244 goto err;
1245 }
1246 ret = synctask_new(this->ctx->env, afr_ta_post_op_do, afr_ta_post_op_done,
1247 ta_frame, local);
1248 if (ret) {
1249 gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
1250 "Failed to launch post-op on thin arbiter for gfid %s",
1251 uuid_utoa(local->inode->gfid));
1252 STACK_DESTROY(ta_frame->root);
1253 goto err;
1254 }
1255
1256 return ret;
1257 err:
1258 afr_changelog_post_op_fail(local->transaction.frame, this, ENOMEM);
1259 return ret;
1260 }
1261
1262 static void
1263 afr_ta_set_fop_state(afr_private_t *priv, afr_local_t *local,
1264 int *on_wire_count)
1265 {
1266 LOCK(&priv->lock);
1267 {
1268 if (priv->release_ta_notify_dom_lock == _gf_true) {
1269 /* Put the fop in waitq until notify dom lock is released.*/
1270 local->fop_state = TA_WAIT_FOR_NOTIFY_LOCK_REL;
1271 list_add_tail(&local->ta_waitq, &priv->ta_waitq);
1272 } else if (priv->ta_bad_child_index == AFR_CHILD_UNKNOWN) {
1273 /* Post-op on thin-arbiter to decide success/failure. */
1274 local->fop_state = TA_GET_INFO_FROM_TA_FILE;
1275 *on_wire_count = ++priv->ta_on_wire_txn_count;
1276 if (*on_wire_count > 1) {
1277 /*Avoid sending multiple on-wire post-ops on TA*/
1278 list_add_tail(&local->ta_onwireq, &priv->ta_onwireq);
1279 }
1280 } else if (local->ta_failed_subvol == priv->ta_bad_child_index) {
1281 /* Post-op on TA not needed as the fop failed on the in-memory bad
1282 * brick. Just mark pending xattrs on the good data brick.*/
1283 local->fop_state = TA_INFO_IN_MEMORY_SUCCESS;
1284 priv->ta_in_mem_txn_count++;
1285 } else {
1286 /* Post-op on TA not needed as the fop succeeded only on the
1287 * in-memory bad data brick and not the good one. Fail the fop.*/
1288 local->fop_state = TA_INFO_IN_MEMORY_FAILED;
1289 priv->ta_in_mem_txn_count++;
1290 }
1291 }
1292 UNLOCK(&priv->lock);
1293 }
1294
1295 static void
1296 afr_ta_fill_failed_subvol(afr_private_t *priv, afr_local_t *local)
1297 {
1298 int i = 0;
1299
1300 for (i = 0; i < priv->child_count; i++) {
1301 if (local->transaction.failed_subvols[i]) {
1302 local->ta_failed_subvol = i;
1303 break;
1304 }
1305 }
1306 }
1307
1308 static void
1309 afr_post_op_handle_success(call_frame_t *frame, xlator_t *this)
1310 {
1311 afr_local_t *local = NULL;
1312
1313 local = frame->local;
1314 if (local->is_new_entry == _gf_true) {
1315 afr_mark_new_entry_changelog(frame, this);
1316 }
1317 afr_changelog_post_op_do(frame, this);
1318
1319 return;
1320 }
1321
1322 static void
1323 afr_post_op_handle_failure(call_frame_t *frame, xlator_t *this, int op_errno)
1324 {
1325 afr_changelog_post_op_fail(frame, this, op_errno);
1326
1327 return;
1328 }
1329
1330 static void
1331 afr_ta_decide_post_op_state(call_frame_t *frame, xlator_t *this)
1332 {
1333 afr_private_t *priv = NULL;
1334 afr_local_t *local = NULL;
1335 int on_wire_count = 0;
1336
1337 priv = this->private;
1338 local = frame->local;
1339
1340 afr_ta_set_fop_state(priv, local, &on_wire_count);
1341
1342 switch (local->fop_state) {
1343 case TA_GET_INFO_FROM_TA_FILE:
1344 if (on_wire_count == 1)
1345 afr_ta_post_op_synctask(this, local);
1346 /*else, fop is queued in ta_onwireq.*/
1347 break;
1348 case TA_WAIT_FOR_NOTIFY_LOCK_REL:
1349 /*Post releasing the notify lock, we will act on this queue*/
1350 break;
1351 case TA_INFO_IN_MEMORY_SUCCESS:
1352 afr_post_op_handle_success(frame, this);
1353 break;
1354 case TA_INFO_IN_MEMORY_FAILED:
1355 afr_post_op_handle_failure(frame, this, EIO);
1356 break;
1357 default:
1358 break;
1359 }
1360 return;
1361 }
1362
1363 static void
1364 afr_handle_failure_using_thin_arbiter(call_frame_t *frame, xlator_t *this)
1365 {
1366 afr_private_t *priv = this->private;
1367 afr_local_t *local = frame->local;
1368
1369 afr_ta_fill_failed_subvol(priv, local);
1370 gf_msg_debug(this->name, 0,
1371 "Fop failed on data brick (%s) for gfid=%s. "
1372 "ta info needed to decide fop result.",
1373 priv->children[local->ta_failed_subvol]->name,
1374 uuid_utoa(local->inode->gfid));
1375 afr_ta_decide_post_op_state(frame, this);
1376 }
1377
1378 static void
1379 afr_changelog_post_op_do(call_frame_t *frame, xlator_t *this)
1380 {
1381 afr_private_t *priv = this->private;
1382 afr_local_t *local = NULL;
1383 dict_t *xattr = NULL;
1384 int i = 0;
1385 int ret = 0;
1386 int idx = 0;
1387 int nothing_failed = 1;
1388 gf_boolean_t need_undirty = _gf_false;
1389 uint32_t hton32_1;
1390
1391 afr_handle_quorum(frame, this);
1392 local = frame->local;
1393 idx = afr_index_for_transaction_type(local->transaction.type);
1394
1395 xattr = dict_new();
1396 if (!xattr) {
1397 afr_changelog_post_op_fail(frame, this, ENOMEM);
1398 goto out;
1399 }
1400
1401 nothing_failed = afr_txn_nothing_failed(frame, this);
1402
1403 if (afr_changelog_pre_op_uninherit(frame, this))
1404 need_undirty = _gf_false;
1405 else
1406 need_undirty = _gf_true;
1407
1408 hton32_1 = htobe32(1);
1409 if (local->op_ret < 0 && !nothing_failed) {
1410 if (afr_need_dirty_marking(frame, this)) {
1411 local->dirty[idx] = hton32_1;
1412 goto set_dirty;
1413 }
1414
1415 afr_changelog_post_op_done(frame, this);
1416 goto out;
1417 }
1418
1419 if (nothing_failed && !need_undirty) {
1420 afr_changelog_post_op_done(frame, this);
1421 goto out;
1422 }
1423
1424 if (local->transaction.in_flight_sb) {
1425 afr_changelog_post_op_fail(frame, this,
1426 local->transaction.in_flight_sb_errno);
1427 goto out;
1428 }
1429
1430 for (i = 0; i < priv->child_count; i++) {
1431 if (local->transaction.failed_subvols[i])
1432 local->pending[i][idx] = hton32_1;
1433 }
1434
1435 ret = afr_set_pending_dict(priv, xattr, local->pending);
1436 if (ret < 0) {
1437 afr_changelog_post_op_fail(frame, this, ENOMEM);
1438 goto out;
1439 }
1440
1441 if (need_undirty)
1442 local->dirty[idx] = htobe32(-1);
1443 else
1444 local->dirty[idx] = 0;
1445
1446 set_dirty:
1447 ret = dict_set_static_bin(xattr, AFR_DIRTY, local->dirty,
1448 sizeof(int) * AFR_NUM_CHANGE_LOGS);
1449 if (ret) {
1450 afr_changelog_post_op_fail(frame, this, ENOMEM);
1451 goto out;
1452 }
1453
1454 afr_changelog_do(frame, this, xattr, afr_changelog_post_op_done,
1455 AFR_TRANSACTION_POST_OP);
1456 out:
1457 if (xattr)
1458 dict_unref(xattr);
1459
1460 return;
1461 }
1462
1463 static int
1464 afr_changelog_post_op_now(call_frame_t *frame, xlator_t *this)
1465 {
1466 afr_private_t *priv = NULL;
1467 afr_local_t *local = NULL;
1468 int failed_count = 0;
1469
1470 priv = this->private;
1471 local = frame->local;
1472
1473 if (priv->thin_arbiter_count) {
1474 failed_count = AFR_COUNT(local->transaction.failed_subvols,
1475 priv->child_count);
1476 if (failed_count == 1) {
1477 afr_handle_failure_using_thin_arbiter(frame, this);
1478 return 0;
1479 } else {
1480 /* Txn either succeeded or failed on both data bricks. Let
1481 * post_op_do handle it as the case might be. */
1482 }
1483 }
1484
1485 afr_changelog_post_op_do(frame, this);
1486 return 0;
1487 }
1488
1489 static gf_boolean_t
1490 afr_changelog_pre_op_uninherit(call_frame_t *frame, xlator_t *this)
1491 {
1492 afr_local_t *local = NULL;
1493 afr_private_t *priv = NULL;
1494 afr_inode_ctx_t *ctx = NULL;
1495 int i = 0;
1496 gf_boolean_t ret = _gf_false;
1497 int type = 0;
1498
1499 local = frame->local;
1500 priv = this->private;
1501 ctx = local->inode_ctx;
1502
1503 type = afr_index_for_transaction_type(local->transaction.type);
1504 if (type != AFR_DATA_TRANSACTION)
1505 return !local->transaction.dirtied;
1506
1507 if (local->transaction.no_uninherit)
1508 return _gf_false;
1509
1510 /* This function must be idempotent. So check if we
1511 were called before and return the same answer again.
1512
1513 It is important to keep this function idempotent for
1514 the call in afr_changelog_post_op_safe() to not have
1515 side effects on the call from afr_changelog_post_op_now()
1516 */
1517 if (local->transaction.uninherit_done)
1518 return local->transaction.uninherit_value;
1519
1520 LOCK(&local->inode->lock);
1521 {
1522 for (i = 0; i < priv->child_count; i++) {
1523 if (local->transaction.pre_op[i] != ctx->pre_op_done[type][i]) {
1524 ret = !local->transaction.dirtied;
1525 goto unlock;
1526 }
1527 }
1528
1529 if (ctx->inherited[type]) {
1530 ret = _gf_true;
1531 ctx->inherited[type]--;
1532 } else if (ctx->on_disk[type]) {
1533 ret = _gf_false;
1534 ctx->on_disk[type]--;
1535 } else {
1536 /* ASSERT */
1537 ret = _gf_false;
1538 }
1539
1540 if (!ctx->inherited[type] && !ctx->on_disk[type]) {
1541 for (i = 0; i < priv->child_count; i++)
1542 ctx->pre_op_done[type][i] = 0;
1543 }
1544 }
1545 unlock:
1546 UNLOCK(&local->inode->lock);
1547
1548 local->transaction.uninherit_done = _gf_true;
1549 local->transaction.uninherit_value = ret;
1550
1551 return ret;
1552 }
1553
1554 static gf_boolean_t
1555 afr_changelog_pre_op_inherit(call_frame_t *frame, xlator_t *this)
1556 {
1557 afr_local_t *local = NULL;
1558 afr_private_t *priv = NULL;
1559 int i = 0;
1560 gf_boolean_t ret = _gf_false;
1561 int type = 0;
1562
1563 local = frame->local;
1564 priv = this->private;
1565
1566 if (local->transaction.type != AFR_DATA_TRANSACTION)
1567 return _gf_false;
1568
1569 type = afr_index_for_transaction_type(local->transaction.type);
1570
1571 LOCK(&local->inode->lock);
1572 {
1573 if (!local->inode_ctx->on_disk[type]) {
1574 /* nothing to inherit yet */
1575 ret = _gf_false;
1576 goto unlock;
1577 }
1578
1579 for (i = 0; i < priv->child_count; i++) {
1580 if (local->transaction.pre_op[i] !=
1581 local->inode_ctx->pre_op_done[type][i]) {
1582 /* either inherit exactly, or don't */
1583 ret = _gf_false;
1584 goto unlock;
1585 }
1586 }
1587
1588 local->inode_ctx->inherited[type]++;
1589
1590 ret = _gf_true;
1591
1592 local->transaction.inherited = _gf_true;
1593 }
1594 unlock:
1595 UNLOCK(&local->inode->lock);
1596
1597 return ret;
1598 }
1599
1600 static gf_boolean_t
1601 afr_changelog_pre_op_update(call_frame_t *frame, xlator_t *this)
1602 {
1603 afr_local_t *local = NULL;
1604 afr_private_t *priv = NULL;
1605 int i = 0;
1606 gf_boolean_t ret = _gf_false;
1607 int type = 0;
1608
1609 local = frame->local;
1610 priv = this->private;
1611
1612 if (local->transaction.type == AFR_ENTRY_TRANSACTION ||
1613 local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION)
1614 return _gf_false;
1615
1616 if (local->transaction.inherited)
1617 /* was already inherited in afr_changelog_pre_op */
1618 return _gf_false;
1619
1620 if (!local->transaction.dirtied)
1621 return _gf_false;
1622
1623 if (!afr_txn_nothing_failed(frame, this))
1624 return _gf_false;
1625
1626 type = afr_index_for_transaction_type(local->transaction.type);
1627
1628 ret = _gf_false;
1629
1630 LOCK(&local->inode->lock);
1631 {
1632 if (!local->inode_ctx->on_disk[type]) {
1633 for (i = 0; i < priv->child_count; i++)
1634 local->inode_ctx->pre_op_done[type][i] =
1635 (!local->transaction.failed_subvols[i]);
1636 } else {
1637 for (i = 0; i < priv->child_count; i++)
1638 if (local->inode_ctx->pre_op_done[type][i] !=
1639 (!local->transaction.failed_subvols[i])) {
1640 local->transaction.no_uninherit = 1;
1641 goto unlock;
1642 }
1643 }
1644 local->inode_ctx->on_disk[type]++;
1645
1646 ret = _gf_true;
1647 }
1648 unlock:
1649 UNLOCK(&local->inode->lock);
1650
1651 return ret;
1652 }
1653
1654 static int
1655 afr_changelog_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
1656 int op_errno, dict_t *xattr, dict_t *xdata)
1657 {
1658 afr_local_t *local = NULL;
1659 int call_count = -1;
1660 int child_index = -1;
1661
1662 local = frame->local;
1663 child_index = (long)cookie;
1664
1665 if (op_ret == -1) {
1666 local->op_errno = op_errno;
1667 afr_transaction_fop_failed(local, child_index);
1668 }
1669
1670 if (xattr)
1671 local->transaction.changelog_xdata[child_index] = dict_ref(xattr);
1672
1673 call_count = afr_frame_return(frame);
1674
1675 if (call_count == 0) {
1676 local->transaction.changelog_resume(frame, this);
1677 }
1678
1679 return 0;
1680 }
1681
1682 static void
1683 afr_changelog_populate_xdata(call_frame_t *frame, afr_xattrop_type_t op,
1684 dict_t **xdata, dict_t **newloc_xdata)
1685 {
1686 int i = 0;
1687 int ret = 0;
1688 char *key = NULL;
1689 int keylen = 0;
1690 const char *name = NULL;
1691 dict_t *xdata1 = NULL;
1692 dict_t *xdata2 = NULL;
1693 xlator_t *this = NULL;
1694 afr_local_t *local = NULL;
1695 afr_private_t *priv = NULL;
1696 gf_boolean_t need_entry_key_set = _gf_true;
1697
1698 local = frame->local;
1699 this = THIS;
1700 priv = this->private;
1701
1702 if (local->transaction.type == AFR_DATA_TRANSACTION ||
1703 local->transaction.type == AFR_METADATA_TRANSACTION)
1704 goto out;
1705
1706 if (!priv->esh_granular)
1707 goto out;
1708
1709 xdata1 = dict_new();
1710 if (!xdata1)
1711 goto out;
1712
1713 name = local->loc.name;
1714 if (local->op == GF_FOP_LINK)
1715 name = local->newloc.name;
1716
1717 switch (op) {
1718 case AFR_TRANSACTION_PRE_OP:
1719 key = GF_XATTROP_ENTRY_IN_KEY;
1720 break;
1721 case AFR_TRANSACTION_POST_OP:
1722 if (afr_txn_nothing_failed(frame, this)) {
1723 key = GF_XATTROP_ENTRY_OUT_KEY;
1724 for (i = 0; i < priv->child_count; i++) {
1725 if (!local->transaction.failed_subvols[i])
1726 continue;
1727 need_entry_key_set = _gf_false;
1728 break;
1729 }
1730 /* If the transaction itself did not fail and there
1731 * are no failed subvolumes, check whether the fop
1732 * failed due to a symmetric error. If it did, do
1733 * not set the ENTRY_OUT xattr which would end up
1734 * deleting a name index which was created possibly by
1735 * an earlier entry txn that may have failed on some
1736 * of the sub-volumes.
1737 */
1738 if (local->op_ret)
1739 need_entry_key_set = _gf_false;
1740 } else {
1741 key = GF_XATTROP_ENTRY_IN_KEY;
1742 }
1743 break;
1744 }
1745
1746 if (need_entry_key_set) {
1747 keylen = strlen(key);
1748 ret = dict_set_strn(xdata1, key, keylen, (char *)name);
1749 if (ret)
1750 gf_msg(THIS->name, GF_LOG_ERROR, 0, AFR_MSG_DICT_SET_FAILED,
1751 "%s/%s: Could not set %s key during xattrop",
1752 uuid_utoa(local->loc.pargfid), local->loc.name, key);
1753 if (local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
1754 xdata2 = dict_new();
1755 if (!xdata2)
1756 goto out;
1757
1758 ret = dict_set_strn(xdata2, key, keylen,
1759 (char *)local->newloc.name);
1760 if (ret)
1761 gf_msg(THIS->name, GF_LOG_ERROR, 0, AFR_MSG_DICT_SET_FAILED,
1762 "%s/%s: Could not set %s key during "
1763 "xattrop",
1764 uuid_utoa(local->newloc.pargfid), local->newloc.name,
1765 key);
1766 }
1767 }
1768
1769 *xdata = xdata1;
1770 *newloc_xdata = xdata2;
1771 xdata1 = xdata2 = NULL;
1772 out:
1773 if (xdata1)
1774 dict_unref(xdata1);
1775 return;
1776 }
1777
1778 static int
1779 afr_changelog_prepare(xlator_t *this, call_frame_t *frame, int *call_count,
1780 afr_changelog_resume_t changelog_resume,
1781 afr_xattrop_type_t op, dict_t **xdata,
1782 dict_t **newloc_xdata)
1783 {
1784 afr_private_t *priv = NULL;
1785 afr_local_t *local = NULL;
1786
1787 local = frame->local;
1788 priv = this->private;
1789
1790 *call_count = afr_changelog_call_count(
1791 local->transaction.type, local->transaction.pre_op,
1792 local->transaction.failed_subvols, priv->child_count);
1793
1794 if (*call_count == 0) {
1795 changelog_resume(frame, this);
1796 return -1;
1797 }
1798
1799 afr_changelog_populate_xdata(frame, op, xdata, newloc_xdata);
1800 local->call_count = *call_count;
1801
1802 local->transaction.changelog_resume = changelog_resume;
1803 return 0;
1804 }
1805
1806 static int
1807 afr_changelog_do(call_frame_t *frame, xlator_t *this, dict_t *xattr,
1808 afr_changelog_resume_t changelog_resume, afr_xattrop_type_t op)
1809 {
1810 afr_local_t *local = NULL;
1811 afr_private_t *priv = NULL;
1812 dict_t *xdata = NULL;
1813 dict_t *newloc_xdata = NULL;
1814 int i = 0;
1815 int call_count = 0;
1816 int ret = 0;
1817
1818 local = frame->local;
1819 priv = this->private;
1820
1821 for (i = 0; i < priv->child_count; i++) {
1822 if (local->transaction.changelog_xdata[i]) {
1823 dict_unref(local->transaction.changelog_xdata[i]);
1824 local->transaction.changelog_xdata[i] = NULL;
1825 }
1826 }
1827
1828 ret = afr_changelog_prepare(this, frame, &call_count, changelog_resume, op,
1829 &xdata, &newloc_xdata);
1830
1831 if (ret)
1832 return 0;
1833
1834 for (i = 0; i < priv->child_count; i++) {
1835 if (!local->transaction.pre_op[i] ||
1836 local->transaction.failed_subvols[i])
1837 continue;
1838
1839 switch (local->transaction.type) {
1840 case AFR_DATA_TRANSACTION:
1841 case AFR_METADATA_TRANSACTION:
1842 if (!local->fd) {
1843 STACK_WIND_COOKIE(
1844 frame, afr_changelog_cbk, (void *)(long)i,
1845 priv->children[i], priv->children[i]->fops->xattrop,
1846 &local->loc, GF_XATTROP_ADD_ARRAY, xattr, xdata);
1847 } else {
1848 STACK_WIND_COOKIE(
1849 frame, afr_changelog_cbk, (void *)(long)i,
1850 priv->children[i], priv->children[i]->fops->fxattrop,
1851 local->fd, GF_XATTROP_ADD_ARRAY, xattr, xdata);
1852 }
1853 break;
1854 case AFR_ENTRY_RENAME_TRANSACTION:
1855
1856 STACK_WIND_COOKIE(frame, afr_changelog_cbk, (void *)(long)i,
1857 priv->children[i],
1858 priv->children[i]->fops->xattrop,
1859 &local->transaction.new_parent_loc,
1860 GF_XATTROP_ADD_ARRAY, xattr, newloc_xdata);
1861 call_count--;
1862
1863 /* fall through */
1864
1865 case AFR_ENTRY_TRANSACTION:
1866 if (local->fd)
1867 STACK_WIND_COOKIE(
1868 frame, afr_changelog_cbk, (void *)(long)i,
1869 priv->children[i], priv->children[i]->fops->fxattrop,
1870 local->fd, GF_XATTROP_ADD_ARRAY, xattr, xdata);
1871 else
1872 STACK_WIND_COOKIE(frame, afr_changelog_cbk, (void *)(long)i,
1873 priv->children[i],
1874 priv->children[i]->fops->xattrop,
1875 &local->transaction.parent_loc,
1876 GF_XATTROP_ADD_ARRAY, xattr, xdata);
1877 break;
1878 }
1879
1880 if (!--call_count)
1881 break;
1882 }
1883
1884 if (xdata)
1885 dict_unref(xdata);
1886 if (newloc_xdata)
1887 dict_unref(newloc_xdata);
1888 return 0;
1889 }
1890
1891 static void
1892 afr_init_optimistic_changelog_for_txn(xlator_t *this, afr_local_t *local)
1893 {
1894 int locked_count = 0;
1895 afr_private_t *priv = NULL;
1896
1897 priv = this->private;
1898
1899 locked_count = AFR_COUNT(local->transaction.pre_op, priv->child_count);
1900 if (priv->optimistic_change_log && locked_count == priv->child_count)
1901 local->optimistic_change_log = 1;
1902
1903 return;
1904 }
1905
1906 int
1907 afr_changelog_pre_op(call_frame_t *frame, xlator_t *this)
1908 {
1909 afr_private_t *priv = this->private;
1910 int i = 0;
1911 int ret = 0;
1912 int call_count = 0;
1913 int op_errno = 0;
1914 afr_local_t *local = NULL;
1915 afr_internal_lock_t *int_lock = NULL;
1916 unsigned char *locked_nodes = NULL;
1917 int idx = -1;
1918 gf_boolean_t pre_nop = _gf_true;
1919 dict_t *xdata_req = NULL;
1920
1921 local = frame->local;
1922 int_lock = &local->internal_lock;
1923 idx = afr_index_for_transaction_type(local->transaction.type);
1924
1925 locked_nodes = afr_locked_nodes_get(local->transaction.type, int_lock);
1926
1927 for (i = 0; i < priv->child_count; i++) {
1928 if (locked_nodes[i]) {
1929 local->transaction.pre_op[i] = 1;
1930 call_count++;
1931 } else {
1932 local->transaction.failed_subvols[i] = 1;
1933 }
1934 }
1935
1936 afr_init_optimistic_changelog_for_txn(this, local);
1937
1938 if (afr_changelog_pre_op_inherit(frame, this))
1939 goto next;
1940
1941 /* This condition should not be met with present code, as
1942 * transaction.done will be called if locks are not acquired on even a
1943 * single node.
1944 */
1945 if (call_count == 0) {
1946 op_errno = ENOTCONN;
1947 goto err;
1948 }
1949
1950 /* Check if the fop can be performed on at least
1951 * quorum number of nodes.
1952 */
1953 if (priv->quorum_count && !afr_has_fop_quorum(frame)) {
1954 op_errno = int_lock->lock_op_errno;
1955 if (op_errno == 0)
1956 op_errno = afr_quorum_errno(priv);
1957 goto err;
1958 }
1959
1960 xdata_req = dict_new();
1961 if (!xdata_req) {
1962 op_errno = ENOMEM;
1963 goto err;
1964 }
1965
1966 if (call_count < priv->child_count)
1967 pre_nop = _gf_false;
1968
1969 /* Set an all-zero pending changelog so that in the cbk, we can get the
1970 * current on-disk values. In a replica 3 volume with arbiter enabled,
1971 * these values are needed to arrive at a go/ no-go of the fop phase to
1972 * avoid ending up in split-brain.*/
1973
1974 ret = afr_set_pending_dict(priv, xdata_req, local->pending);
1975 if (ret < 0) {
1976 op_errno = ENOMEM;
1977 goto err;
1978 }
1979
1980 if (afr_needs_changelog_update(local)) {
1981 local->dirty[idx] = htobe32(1);
1982
1983 ret = dict_set_static_bin(xdata_req, AFR_DIRTY, local->dirty,
1984 sizeof(int) * AFR_NUM_CHANGE_LOGS);
1985 if (ret) {
1986 op_errno = ENOMEM;
1987 goto err;
1988 }
1989
1990 pre_nop = _gf_false;
1991 local->transaction.dirtied = 1;
1992 }
1993
1994 if (pre_nop)
1995 goto next;
1996
1997 if (!local->pre_op_compat) {
1998 dict_copy(xdata_req, local->xdata_req);
1999 goto next;
2000 }
2001
2002 afr_changelog_do(frame, this, xdata_req, afr_transaction_perform_fop,
2003 AFR_TRANSACTION_PRE_OP);
2004
2005 if (xdata_req)
2006 dict_unref(xdata_req);
2007
2008 return 0;
2009 next:
2010 afr_transaction_perform_fop(frame, this);
2011
2012 if (xdata_req)
2013 dict_unref(xdata_req);
2014
2015 return 0;
2016 err:
2017 local->internal_lock.lock_cbk = afr_transaction_done;
2018 local->op_ret = -1;
2019 local->op_errno = op_errno;
2020
2021 afr_handle_lock_acquire_failure(local);
2022
2023 if (xdata_req)
2024 dict_unref(xdata_req);
2025
2026 return 0;
2027 }
2028
2029 static int
2030 afr_post_nonblocking_lock_cbk(call_frame_t *frame, xlator_t *this)
2031 {
2032 afr_internal_lock_t *int_lock = NULL;
2033 afr_local_t *local = NULL;
2034
2035 local = frame->local;
2036 int_lock = &local->internal_lock;
2037
2038 /* Initiate blocking locks if non-blocking has failed */
2039 if (int_lock->lock_op_ret < 0) {
2040 gf_msg_debug(this->name, 0,
2041 "Non blocking locks failed. Proceeding to blocking");
2042 int_lock->lock_cbk = afr_internal_lock_finish;
2043 afr_blocking_lock(frame, this);
2044 } else {
2045 gf_msg_debug(this->name, 0,
2046 "Non blocking locks done. Proceeding to FOP");
2047
2048 afr_internal_lock_finish(frame, this);
2049 }
2050
2051 return 0;
2052 }
2053
2054 static int
2055 afr_set_transaction_flock(xlator_t *this, afr_local_t *local,
2056 afr_lockee_t *lockee)
2057 {
2058 afr_private_t *priv = NULL;
2059 struct gf_flock *flock = NULL;
2060
2061 priv = this->private;
2062 flock = &lockee->flock;
2063
2064 if ((priv->arbiter_count || local->transaction.eager_lock_on ||
2065 priv->full_lock) &&
2066 local->transaction.type == AFR_DATA_TRANSACTION) {
2067 /*Lock entire file to avoid network split brains.*/
2068 flock->l_len = 0;
2069 flock->l_start = 0;
2070 } else {
2071 flock->l_len = local->transaction.len;
2072 flock->l_start = local->transaction.start;
2073 }
2074 flock->l_type = F_WRLCK;
2075
2076 return 0;
2077 }
2078
2079 int
2080 afr_lock(call_frame_t *frame, xlator_t *this)
2081 {
2082 afr_internal_lock_t *int_lock = NULL;
2083 afr_local_t *local = NULL;
2084 int i = 0;
2085
2086 local = frame->local;
2087 int_lock = &local->internal_lock;
2088
2089 int_lock->lock_cbk = afr_post_nonblocking_lock_cbk;
2090 int_lock->domain = this->name;
2091
2092 switch (local->transaction.type) {
2093 case AFR_DATA_TRANSACTION:
2094 case AFR_METADATA_TRANSACTION:
2095 for (i = 0; i < int_lock->lockee_count; i++) {
2096 afr_set_transaction_flock(this, local, &int_lock->lockee[i]);
2097 }
2098
2099 break;
2100
2101 case AFR_ENTRY_TRANSACTION:
2102 if (local->transaction.parent_loc.path)
2103 int_lock->lk_loc = &local->transaction.parent_loc;
2104 else
2105 GF_ASSERT(local->fd);
2106 break;
2107 case AFR_ENTRY_RENAME_TRANSACTION:
2108 break;
2109 }
2110 afr_lock_nonblocking(frame, this);
2111
2112 return 0;
2113 }
2114
2115 static gf_boolean_t
2116 afr_locals_overlap(afr_local_t *local1, afr_local_t *local2)
2117 {
2118 uint64_t start1 = local1->transaction.start;
2119 uint64_t start2 = local2->transaction.start;
2120 uint64_t end1 = 0;
2121 uint64_t end2 = 0;
2122
2123 if (local1->transaction.len)
2124 end1 = start1 + local1->transaction.len - 1;
2125 else
2126 end1 = ULLONG_MAX;
2127
2128 if (local2->transaction.len)
2129 end2 = start2 + local2->transaction.len - 1;
2130 else
2131 end2 = ULLONG_MAX;
2132
2133 return ((end1 >= start2) && (end2 >= start1));
2134 }
2135
2136 gf_boolean_t
2137 afr_has_lock_conflict(afr_local_t *local, gf_boolean_t waitlist_check)
2138 {
2139 afr_local_t *each = NULL;
2140 afr_lock_t *lock = NULL;
2141
2142 lock = &local->inode_ctx->lock[local->transaction.type];
2143 /*
2144 * Once full file lock is acquired in eager-lock phase, overlapping
2145 * writes do not compete for inode-locks, instead are transferred to the
2146 * next writes. Because of this overlapping writes are not ordered.
2147 * This can cause inconsistencies in replication.
2148 * Example:
2149 * Two overlapping writes w1, w2 are sent in parallel on same fd
2150 * in two threads t1, t2.
2151 * Both threads can execute afr_writev_wind in the following manner.
2152 * t1 winds w1 on brick-0
2153 * t2 winds w2 on brick-0
2154 * t2 winds w2 on brick-1
2155 * t1 winds w1 on brick-1
2156 *
2157 * This check makes sure the locks are not transferred for
2158 * overlapping writes.
2159 */
2160 list_for_each_entry(each, &lock->owners, transaction.owner_list)
2161 {
2162 if (afr_locals_overlap(each, local)) {
2163 return _gf_true;
2164 }
2165 }
2166
2167 if (!waitlist_check)
2168 return _gf_false;
2169 list_for_each_entry(each, &lock->waiting, transaction.wait_list)
2170 {
2171 if (afr_locals_overlap(each, local)) {
2172 return _gf_true;
2173 }
2174 }
2175 return _gf_false;
2176 }
2177
2178 /* }}} */
2179 static void
2180 afr_copy_inodelk_vars(afr_internal_lock_t *dst, afr_internal_lock_t *src,
2181 xlator_t *this, int lockee_num)
2182 {
2183 afr_private_t *priv = this->private;
2184 afr_lockee_t *sl = &src->lockee[lockee_num];
2185 afr_lockee_t *dl = &dst->lockee[lockee_num];
2186
2187 dst->domain = src->domain;
2188 dl->flock.l_len = sl->flock.l_len;
2189 dl->flock.l_start = sl->flock.l_start;
2190 dl->flock.l_type = sl->flock.l_type;
2191 dl->locked_count = sl->locked_count;
2192 memcpy(dl->locked_nodes, sl->locked_nodes,
2193 priv->child_count * sizeof(*dl->locked_nodes));
2194 }
2195
2196 static void
2197 __afr_transaction_wake_shared(afr_local_t *local, struct list_head *shared)
2198 {
2199 gf_boolean_t conflict = _gf_false;
2200 afr_local_t *each = NULL;
2201 afr_lock_t *lock = &local->inode_ctx->lock[local->transaction.type];
2202
2203 while (!conflict) {
2204 if (list_empty(&lock->waiting))
2205 return;
2206 each = list_entry(lock->waiting.next, afr_local_t,
2207 transaction.wait_list);
2208 if (afr_has_lock_conflict(each, _gf_false)) {
2209 conflict = _gf_true;
2210 }
2211 if (conflict && !list_empty(&lock->owners))
2212 return;
2213 afr_copy_inodelk_vars(&each->internal_lock, &local->internal_lock,
2214 each->transaction.frame->this, 0);
2215 list_move_tail(&each->transaction.wait_list, shared);
2216 list_add_tail(&each->transaction.owner_list, &lock->owners);
2217 }
2218 }
2219
2220 static void
2221 afr_lock_resume_shared(struct list_head *list)
2222 {
2223 afr_local_t *each = NULL;
2224
2225 while (!list_empty(list)) {
2226 each = list_entry(list->next, afr_local_t, transaction.wait_list);
2227 list_del_init(&each->transaction.wait_list);
2228 afr_changelog_pre_op(each->transaction.frame,
2229 each->transaction.frame->this);
2230 }
2231 }
2232
2233 static int
2234 afr_internal_lock_finish(call_frame_t *frame, xlator_t *this)
2235 {
2236 afr_local_t *local = frame->local;
2237 afr_lock_t *lock = NULL;
2238
2239 local->internal_lock.lock_cbk = NULL;
2240 if (!local->transaction.eager_lock_on) {
2241 if (local->internal_lock.lock_op_ret < 0) {
2242 afr_transaction_done(frame, this);
2243 return 0;
2244 }
2245 afr_changelog_pre_op(frame, this);
2246 } else {
2247 lock = &local->inode_ctx->lock[local->transaction.type];
2248 if (local->internal_lock.lock_op_ret < 0) {
2249 afr_handle_lock_acquire_failure(local);
2250 } else {
2251 lock->event_generation = local->event_generation;
2252 afr_changelog_pre_op(frame, this);
2253 }
2254 }
2255
2256 return 0;
2257 }
2258
2259 static gf_boolean_t
2260 afr_are_conflicting_ops_waiting(afr_local_t *local, xlator_t *this)
2261 {
2262 afr_lock_t *lock = NULL;
2263 lock = &local->inode_ctx->lock[local->transaction.type];
2264
2265 /* Lets say mount1 has eager-lock(full-lock) and after the eager-lock
2266 * is taken mount2 opened the same file, it won't be able to
2267 * perform any {meta,}data operations until mount1 releases eager-lock.
2268 * To avoid such scenario do not enable eager-lock for this transaction
2269 * if open-fd-count is > 1 for metadata transactions and if num-inodelks > 1
2270 * for data transactions
2271 */
2272
2273 if (local->transaction.type == AFR_METADATA_TRANSACTION) {
2274 if (local->inode_ctx->open_fd_count > 1) {
2275 return _gf_true;
2276 }
2277 } else if (local->transaction.type == AFR_DATA_TRANSACTION) {
2278 if (lock->num_inodelks > 1) {
2279 return _gf_true;
2280 }
2281 }
2282
2283 return _gf_false;
2284 }
2285
2286 static gf_boolean_t
2287 afr_is_delayed_changelog_post_op_needed(call_frame_t *frame, xlator_t *this,
2288 int delay)
2289 {
2290 afr_local_t *local = NULL;
2291 afr_lock_t *lock = NULL;
2292 gf_boolean_t res = _gf_false;
2293
2294 local = frame->local;
2295 lock = &local->inode_ctx->lock[local->transaction.type];
2296
2297 if (!afr_txn_nothing_failed(frame, this)) {
2298 lock->release = _gf_true;
2299 goto out;
2300 }
2301
2302 if (afr_are_conflicting_ops_waiting(local, this)) {
2303 lock->release = _gf_true;
2304 goto out;
2305 }
2306
2307 if (!list_empty(&lock->owners))
2308 goto out;
2309 else
2310 GF_ASSERT(list_empty(&lock->waiting));
2311
2312 if (lock->release) {
2313 goto out;
2314 }
2315
2316 if (!delay) {
2317 goto out;
2318 }
2319
2320 if (local->transaction.disable_delayed_post_op) {
2321 goto out;
2322 }
2323
2324 if ((local->op != GF_FOP_WRITE) && (local->op != GF_FOP_FXATTROP) &&
2325 (local->op != GF_FOP_FSYNC)) {
2326 /*Only allow writes/fsyncs but shard does [f]xattrops on writes, so
2327 * they are fine too*/
2328 goto out;
2329 }
2330
2331 res = _gf_true;
2332 out:
2333 return res;
2334 }
2335
2336 void
2337 afr_delayed_changelog_wake_up_cbk(void *data)
2338 {
2339 afr_lock_t *lock = NULL;
2340 afr_local_t *local = data;
2341 afr_local_t *timer_local = NULL;
2342 struct list_head shared;
2343
2344 INIT_LIST_HEAD(&shared);
2345 lock = &local->inode_ctx->lock[local->transaction.type];
2346 LOCK(&local->inode->lock);
2347 {
2348 timer_local = list_entry(lock->post_op.next, afr_local_t,
2349 transaction.owner_list);
2350 if (list_empty(&lock->owners) && (local == timer_local)) {
2351 GF_ASSERT(list_empty(&lock->waiting));
2352 /*Last owner*/
2353 lock->release = _gf_true;
2354 lock->delay_timer = NULL;
2355 }
2356 }
2357 UNLOCK(&local->inode->lock);
2358 afr_changelog_post_op_now(local->transaction.frame,
2359 local->transaction.frame->this);
2360 }
2361
2362 /* SET operation */
2363 int
2364 afr_fd_report_unstable_write(xlator_t *this, afr_local_t *local)
2365 {
2366 LOCK(&local->inode->lock);
2367 {
2368 local->inode_ctx->witnessed_unstable_write = _gf_true;
2369 }
2370 UNLOCK(&local->inode->lock);
2371
2372 return 0;
2373 }
2374
2375 /* TEST and CLEAR operation */
2376 gf_boolean_t
2377 afr_fd_has_witnessed_unstable_write(xlator_t *this, inode_t *inode)
2378 {
2379 afr_inode_ctx_t *ctx = NULL;
2380 gf_boolean_t witness = _gf_false;
2381
2382 LOCK(&inode->lock);
2383 {
2384 ctx = __afr_inode_ctx_get(this, inode);
2385 if (ctx == NULL)
2386 goto unlock;
2387
2388 if (ctx->witnessed_unstable_write) {
2389 witness = _gf_true;
2390 ctx->witnessed_unstable_write = _gf_false;
2391 }
2392 }
2393 unlock:
2394 UNLOCK(&inode->lock);
2395
2396 return witness;
2397 }
2398
2399 static int
2400 afr_changelog_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
2401 int op_ret, int op_errno, struct iatt *pre,
2402 struct iatt *post, dict_t *xdata)
2403 {
2404 afr_private_t *priv = NULL;
2405 int child_index = (long)cookie;
2406 int call_count = -1;
2407 afr_local_t *local = NULL;
2408
2409 priv = this->private;
2410 local = frame->local;
2411
2412 if (op_ret != 0) {
2413 /* Failure of fsync() is as good as failure of previous
2414 write(). So treat it like one.
2415 */
2416 gf_msg(this->name, GF_LOG_WARNING, op_errno, AFR_MSG_FSYNC_FAILED,
2417 "fsync(%s) failed on subvolume %s. Transaction was %s",
2418 uuid_utoa(local->fd->inode->gfid),
2419 priv->children[child_index]->name, gf_fop_list[local->op]);
2420
2421 afr_transaction_fop_failed(local, child_index);
2422 }
2423
2424 call_count = afr_frame_return(frame);
2425
2426 if (call_count == 0)
2427 afr_changelog_post_op_now(frame, this);
2428
2429 return 0;
2430 }
2431
2432 static int
2433 afr_changelog_fsync(call_frame_t *frame, xlator_t *this)
2434 {
2435 afr_local_t *local = NULL;
2436 int i = 0;
2437 int call_count = 0;
2438 afr_private_t *priv = NULL;
2439 dict_t *xdata = NULL;
2440 GF_UNUSED int ret = -1;
2441
2442 local = frame->local;
2443 priv = this->private;
2444
2445 call_count = AFR_COUNT(local->transaction.pre_op, priv->child_count);
2446
2447 if (!call_count) {
2448 /* will go straight to unlock */
2449 afr_changelog_post_op_now(frame, this);
2450 return 0;
2451 }
2452
2453 local->call_count = call_count;
2454
2455 xdata = dict_new();
2456 if (xdata) {
2457 ret = dict_set_int32_sizen(xdata, "batch-fsync", 1);
2458 ret = dict_set_str(xdata, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
2459 }
2460
2461 for (i = 0; i < priv->child_count; i++) {
2462 if (!local->transaction.pre_op[i])
2463 continue;
2464
2465 STACK_WIND_COOKIE(frame, afr_changelog_fsync_cbk, (void *)(long)i,
2466 priv->children[i], priv->children[i]->fops->fsync,
2467 local->fd, 1, xdata);
2468 if (!--call_count)
2469 break;
2470 }
2471
2472 if (xdata)
2473 dict_unref(xdata);
2474
2475 return 0;
2476 }
2477
2478 static int
2479 afr_changelog_post_op_safe(call_frame_t *frame, xlator_t *this)
2480 {
2481 afr_local_t *local = NULL;
2482 afr_private_t *priv = NULL;
2483
2484 local = frame->local;
2485 priv = this->private;
2486
2487 if (!local->fd || local->transaction.type != AFR_DATA_TRANSACTION) {
2488 afr_changelog_post_op_now(frame, this);
2489 return 0;
2490 }
2491
2492 if (afr_changelog_pre_op_uninherit(frame, this) &&
2493 afr_txn_nothing_failed(frame, this)) {
2494 /* just detected that this post-op is about to
2495 be optimized away as a new write() has
2496 already piggybacked on this frame's changelog.
2497 */
2498 afr_changelog_post_op_now(frame, this);
2499 return 0;
2500 }
2501
2502 /* Calling afr_changelog_post_op_now() now will result in
2503 issuing ->[f]xattrop().
2504
2505 Performing a hard POST-OP (->[f]xattrop() FOP) is a more
2506 responsible operation that what it might appear on the surface.
2507
2508 The changelog of a file (in the xattr of the file on the server)
2509 stores information (pending count) about the state of the file
2510 on the OTHER server. This changelog is blindly trusted, and must
2511 therefore be updated in such a way it remains trustworthy. This
2512 implies that decrementing the pending count (essentially "clearing
2513 the dirty flag") must be done STRICTLY after we are sure that the
2514 operation on the other server has reached stable storage.
2515
2516 While the backend filesystem on that server will eventually flush
2517 it to stable storage, we (being in userspace) have no mechanism
2518 to get notified when the write became "stable".
2519
2520 This means we need take matter into our own hands and issue an
2521 fsync() EVEN IF THE APPLICATION WAS PERFORMING UNSTABLE WRITES,
2522 and get an acknowledgement for it. And we need to wait for the
2523 fsync() acknowledgement before initiating the hard POST-OP.
2524
2525 However if the FD itself was opened in O_SYNC or O_DSYNC then
2526 we are already guaranteed that the writes were made stable as
2527 part of the FOP itself. The same holds true for NFS stable
2528 writes which happen on an anonymous FD with O_DSYNC or O_SYNC
2529 flag set in the writev() @flags param. For all other write types,
2530 mark a flag in the fdctx whenever an unstable write is witnessed.
2531 */
2532
2533 if (!afr_fd_has_witnessed_unstable_write(this, local->inode)) {
2534 afr_changelog_post_op_now(frame, this);
2535 return 0;
2536 }
2537
2538 /* Check whether users want durability and perform fsync/post-op
2539 * accordingly.
2540 */
2541 if (priv->ensure_durability) {
2542 /* Time to fsync() */
2543 afr_changelog_fsync(frame, this);
2544 } else {
2545 afr_changelog_post_op_now(frame, this);
2546 }
2547
2548 return 0;
2549 }
2550
2551 void
2552 afr_changelog_post_op(call_frame_t *frame, xlator_t *this)
2553 {
2554 struct timespec delta = {
2555 0,
2556 };
2557 afr_private_t *priv = NULL;
2558 afr_local_t *local = frame->local;
2559 afr_lock_t *lock = NULL;
2560 gf_boolean_t post_op = _gf_true;
2561 struct list_head shared;
2562
2563 priv = this->private;
2564 delta.tv_sec = priv->post_op_delay_secs;
2565 delta.tv_nsec = 0;
2566
2567 INIT_LIST_HEAD(&shared);
2568 if (!local->transaction.eager_lock_on)
2569 goto out;
2570
2571 lock = &local->inode_ctx->lock[local->transaction.type];
2572 LOCK(&local->inode->lock);
2573 {
2574 list_del_init(&local->transaction.owner_list);
2575 list_add(&local->transaction.owner_list, &lock->post_op);
2576 __afr_transaction_wake_shared(local, &shared);
2577
2578 if (!afr_is_delayed_changelog_post_op_needed(frame, this,
2579 delta.tv_sec)) {
2580 if (list_empty(&lock->owners))
2581 lock->release = _gf_true;
2582 goto unlock;
2583 }
2584
2585 GF_ASSERT(lock->delay_timer == NULL);
2586 lock->delay_timer = gf_timer_call_after(
2587 this->ctx, delta, afr_delayed_changelog_wake_up_cbk, local);
2588 if (!lock->delay_timer) {
2589 lock->release = _gf_true;
2590 } else {
2591 post_op = _gf_false;
2592 }
2593 }
2594 unlock:
2595 UNLOCK(&local->inode->lock);
2596
2597 if (!list_empty(&shared)) {
2598 afr_lock_resume_shared(&shared);
2599 }
2600
2601 out:
2602 if (post_op) {
2603 if (!local->transaction.eager_lock_on || lock->release) {
2604 afr_changelog_post_op_safe(frame, this);
2605 } else {
2606 afr_changelog_post_op_now(frame, this);
2607 }
2608 }
2609 }
2610
2611 int
2612 afr_transaction_resume(call_frame_t *frame, xlator_t *this)
2613 {
2614 afr_local_t *local = NULL;
2615
2616 local = frame->local;
2617
2618 afr_restore_lk_owner(frame);
2619
2620 afr_handle_symmetric_errors(frame, this);
2621
2622 if (!local->pre_op_compat)
2623 /* new mode, pre-op was done along
2624 with OP */
2625 afr_changelog_pre_op_update(frame, this);
2626
2627 afr_changelog_post_op(frame, this);
2628
2629 return 0;
2630 }
2631
2632 /**
2633 * afr_transaction_fop_failed - inform that an fop failed
2634 */
2635
2636 void
2637 afr_transaction_fop_failed(afr_local_t *local, int child_index)
2638 {
2639 local->transaction.failed_subvols[child_index] = 1;
2640 }
2641
2642 static gf_boolean_t
2643 __need_previous_lock_unlocked(afr_local_t *local)
2644 {
2645 afr_lock_t *lock = NULL;
2646
2647 lock = &local->inode_ctx->lock[local->transaction.type];
2648 if (!lock->acquired)
2649 return _gf_false;
2650 if (lock->acquired && lock->event_generation != local->event_generation)
2651 return _gf_true;
2652 return _gf_false;
2653 }
2654
2655 static void
2656 __afr_eager_lock_handle(afr_local_t *local, gf_boolean_t *take_lock,
2657 gf_boolean_t *do_pre_op, afr_local_t **timer_local)
2658 {
2659 afr_lock_t *lock = NULL;
2660 afr_local_t *owner_local = NULL;
2661 xlator_t *this = local->transaction.frame->this;
2662
2663 local->transaction.eager_lock_on = _gf_true;
2664 afr_set_lk_owner(local->transaction.frame, this, local->inode);
2665
2666 lock = &local->inode_ctx->lock[local->transaction.type];
2667 if (__need_previous_lock_unlocked(local)) {
2668 if (!list_empty(&lock->owners)) {
2669 lock->release = _gf_true;
2670 } else if (lock->delay_timer) {
2671 lock->release = _gf_true;
2672 if (gf_timer_call_cancel(this->ctx, lock->delay_timer)) {
2673 /* It will be put in frozen list
2674 * in the code flow below*/
2675 } else {
2676 *timer_local = list_entry(lock->post_op.next, afr_local_t,
2677 transaction.owner_list);
2678 lock->delay_timer = NULL;
2679 }
2680 }
2681 }
2682
2683 if (lock->release) {
2684 list_add_tail(&local->transaction.wait_list, &lock->frozen);
2685 *take_lock = _gf_false;
2686 goto out;
2687 }
2688
2689 if (lock->delay_timer) {
2690 *take_lock = _gf_false;
2691 if (gf_timer_call_cancel(this->ctx, lock->delay_timer)) {
2692 list_add_tail(&local->transaction.wait_list, &lock->frozen);
2693 } else {
2694 *timer_local = list_entry(lock->post_op.next, afr_local_t,
2695 transaction.owner_list);
2696 afr_copy_inodelk_vars(&local->internal_lock,
2697 &(*timer_local)->internal_lock, this, 0);
2698 lock->delay_timer = NULL;
2699 *do_pre_op = _gf_true;
2700 list_add_tail(&local->transaction.owner_list, &lock->owners);
2701 }
2702 goto out;
2703 }
2704
2705 if (!list_empty(&lock->owners)) {
2706 if (!lock->acquired || afr_has_lock_conflict(local, _gf_true)) {
2707 list_add_tail(&local->transaction.wait_list, &lock->waiting);
2708 *take_lock = _gf_false;
2709 goto out;
2710 }
2711 owner_local = list_entry(lock->owners.next, afr_local_t,
2712 transaction.owner_list);
2713 afr_copy_inodelk_vars(&local->internal_lock,
2714 &owner_local->internal_lock, this, 0);
2715 *take_lock = _gf_false;
2716 *do_pre_op = _gf_true;
2717 }
2718
2719 if (lock->acquired)
2720 GF_ASSERT(!(*take_lock));
2721 list_add_tail(&local->transaction.owner_list, &lock->owners);
2722 out:
2723 return;
2724 }
2725
2726 static void
2727 afr_transaction_start(afr_local_t *local, xlator_t *this)
2728 {
2729 afr_private_t *priv = NULL;
2730 gf_boolean_t take_lock = _gf_true;
2731 gf_boolean_t do_pre_op = _gf_false;
2732 afr_local_t *timer_local = NULL;
2733
2734 priv = this->private;
2735
2736 if (local->transaction.type != AFR_DATA_TRANSACTION &&
2737 local->transaction.type != AFR_METADATA_TRANSACTION)
2738 goto lock_phase;
2739
2740 if (!priv->eager_lock)
2741 goto lock_phase;
2742
2743 LOCK(&local->inode->lock);
2744 {
2745 __afr_eager_lock_handle(local, &take_lock, &do_pre_op, &timer_local);
2746 }
2747 UNLOCK(&local->inode->lock);
2748 lock_phase:
2749 if (!local->transaction.eager_lock_on) {
2750 afr_set_lk_owner(local->transaction.frame, this,
2751 local->transaction.frame->root);
2752 }
2753
2754 if (take_lock) {
2755 afr_lock(local->transaction.frame, this);
2756 } else if (do_pre_op) {
2757 afr_changelog_pre_op(local->transaction.frame, this);
2758 }
2759 /*Always call delayed_changelog_wake_up_cbk after calling pre-op above
2760 * so that any inheriting can happen*/
2761 if (timer_local)
2762 afr_delayed_changelog_wake_up_cbk(timer_local);
2763 }
2764
2765 static int
2766 afr_write_txn_refresh_done(call_frame_t *frame, xlator_t *this, int err)
2767 {
2768 afr_local_t *local = frame->local;
2769
2770 if (err) {
2771 AFR_SET_ERROR_AND_CHECK_SPLIT_BRAIN(-1, err);
2772 goto fail;
2773 }
2774
2775 afr_transaction_start(local, this);
2776 return 0;
2777 fail:
2778 local->transaction.unwind(frame, this);
2779 AFR_STACK_DESTROY(frame);
2780 return 0;
2781 }
2782
2783 static int
2784 afr_transaction_lockee_init(call_frame_t *frame)
2785 {
2786 afr_local_t *local = frame->local;
2787 afr_internal_lock_t *int_lock = &local->internal_lock;
2788 afr_private_t *priv = frame->this->private;
2789 int ret = 0;
2790
2791 switch (local->transaction.type) {
2792 case AFR_DATA_TRANSACTION:
2793 case AFR_METADATA_TRANSACTION:
2794 ret = afr_add_inode_lockee(local, priv->child_count);
2795 break;
2796
2797 case AFR_ENTRY_TRANSACTION:
2798 case AFR_ENTRY_RENAME_TRANSACTION:
2799 ret = afr_add_entry_lockee(local, &local->transaction.parent_loc,
2800 local->transaction.basename,
2801 priv->child_count);
2802 if (ret) {
2803 goto out;
2804 }
2805 if (local->op == GF_FOP_RENAME) {
2806 ret = afr_add_entry_lockee(
2807 local, &local->transaction.new_parent_loc,
2808 local->transaction.new_basename, priv->child_count);
2809 if (ret) {
2810 goto out;
2811 }
2812
2813 if (local->newloc.inode &&
2814 IA_ISDIR(local->newloc.inode->ia_type)) {
2815 ret = afr_add_entry_lockee(local, &local->newloc, NULL,
2816 priv->child_count);
2817 if (ret) {
2818 goto out;
2819 }
2820 }
2821 } else if (local->op == GF_FOP_RMDIR) {
2822 ret = afr_add_entry_lockee(local, &local->loc, NULL,
2823 priv->child_count);
2824 if (ret) {
2825 goto out;
2826 }
2827 }
2828
2829 if (int_lock->lockee_count > 1) {
2830 qsort(int_lock->lockee, int_lock->lockee_count,
2831 sizeof(*int_lock->lockee), afr_entry_lockee_cmp);
2832 }
2833 break;
2834 }
2835 out:
2836 return ret;
2837 }
2838
2839 int
2840 afr_transaction(call_frame_t *frame, xlator_t *this, afr_transaction_type type)
2841 {
2842 afr_local_t *local = NULL;
2843 afr_private_t *priv = NULL;
2844 int ret = -1;
2845 int event_generation = 0;
2846
2847 local = frame->local;
2848 priv = this->private;
2849 local->transaction.frame = frame;
2850
2851 local->transaction.type = type;
2852
2853 if (priv->quorum_count && !afr_has_quorum(local->child_up, priv, NULL)) {
2854 ret = -afr_quorum_errno(priv);
2855 goto out;
2856 }
2857
2858 if (!afr_is_consistent_io_possible(local, priv, &ret)) {
2859 ret = -ret; /*op_errno to ret conversion*/
2860 goto out;
2861 }
2862
2863 if (priv->thin_arbiter_count && !afr_ta_has_quorum(priv, local)) {
2864 ret = -afr_quorum_errno(priv);
2865 goto out;
2866 }
2867
2868 ret = afr_transaction_local_init(local, this);
2869 if (ret < 0)
2870 goto out;
2871
2872 ret = afr_transaction_lockee_init(frame);
2873 if (ret)
2874 goto out;
2875
2876 if (type != AFR_METADATA_TRANSACTION) {
2877 goto txn_start;
2878 }
2879
2880 ret = afr_inode_get_readable(frame, local->inode, this, local->readable,
2881 &event_generation, type);
2882 if (ret < 0 ||
2883 afr_is_inode_refresh_reqd(local->inode, this, priv->event_generation,
2884 event_generation)) {
2885 afr_inode_refresh(frame, this, local->inode, local->loc.gfid,
2886 afr_write_txn_refresh_done);
2887 ret = 0;
2888 goto out;
2889 }
2890
2891 txn_start:
2892 ret = 0;
2893 afr_transaction_start(local, this);
2894 out:
2895 return ret;
2896 }