Debian Codegraph @

xlators/cluster/afr/src/afr-read-txn.c

raw
1 /*
2 Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
3 This file is part of GlusterFS.
4
5 This file is licensed to you under your choice of the GNU Lesser
6 General Public License, version 3 or any later version (LGPLv3 or
7 later), or the GNU General Public License, version 2 (GPLv2), in all
8 cases as published by the Free Software Foundation.
9 */
10
11 #include "afr-transaction.h"
12 #include "afr-messages.h"
13
14 void
15 afr_pending_read_increment(afr_private_t *priv, int child_index)
16 {
17 if (child_index < 0 || child_index > priv->child_count)
18 return;
19
20 GF_ATOMIC_INC(priv->pending_reads[child_index]);
21 }
22
23 void
24 afr_pending_read_decrement(afr_private_t *priv, int child_index)
25 {
26 if (child_index < 0 || child_index > priv->child_count)
27 return;
28
29 GF_ATOMIC_DEC(priv->pending_reads[child_index]);
30 }
31
32 void
33 afr_read_txn_wind(call_frame_t *frame, xlator_t *this, int subvol)
34 {
35 afr_local_t *local = NULL;
36 afr_private_t *priv = NULL;
37
38 local = frame->local;
39 priv = this->private;
40
41 afr_pending_read_decrement(priv, local->read_subvol);
42 local->read_subvol = subvol;
43 afr_pending_read_increment(priv, subvol);
44 local->readfn(frame, this, subvol);
45 }
46
47 static int
48 afr_read_txn_next_subvol(call_frame_t *frame, xlator_t *this)
49 {
50 afr_local_t *local = NULL;
51 afr_private_t *priv = NULL;
52 int i = 0;
53 int subvol = -1;
54
55 local = frame->local;
56 priv = this->private;
57
58 for (i = 0; i < priv->child_count; i++) {
59 if (!local->readable[i]) {
60 /* don't even bother trying here.
61 just mark as attempted and move on. */
62 local->read_attempted[i] = 1;
63 continue;
64 }
65
66 if (!local->read_attempted[i]) {
67 subvol = i;
68 break;
69 }
70 }
71
72 /* If no more subvols were available for reading, we leave
73 @subvol as -1, which is an indication we have run out of
74 readable subvols. */
75 if (subvol != -1)
76 local->read_attempted[subvol] = 1;
77 afr_read_txn_wind(frame, this, subvol);
78
79 return 0;
80 }
81
82 static int
83 afr_ta_read_txn_done(int ret, call_frame_t *ta_frame, void *opaque)
84 {
85 STACK_DESTROY(ta_frame->root);
86 return 0;
87 }
88
89 static int
90 afr_ta_read_txn(void *opaque)
91 {
92 call_frame_t *frame = NULL;
93 xlator_t *this = NULL;
94 int read_subvol = -1;
95 int query_child = AFR_CHILD_UNKNOWN;
96 int possible_bad_child = AFR_CHILD_UNKNOWN;
97 int ret = 0;
98 int op_errno = ENOMEM;
99 afr_local_t *local = NULL;
100 afr_private_t *priv = NULL;
101 struct gf_flock flock = {
102 0,
103 };
104 dict_t *xdata_req = NULL;
105 dict_t *xdata_rsp = NULL;
106 int **pending = NULL;
107 loc_t loc = {
108 0,
109 };
110
111 frame = (call_frame_t *)opaque;
112 this = frame->this;
113 local = frame->local;
114 priv = this->private;
115 query_child = local->read_txn_query_child;
116
117 if (query_child == AFR_CHILD_ZERO) {
118 possible_bad_child = AFR_CHILD_ONE;
119 } else if (query_child == AFR_CHILD_ONE) {
120 possible_bad_child = AFR_CHILD_ZERO;
121 } else {
122 /*read_txn_query_child is AFR_CHILD_UNKNOWN*/
123 goto out;
124 }
125
126 /* Ask the query_child to see if it blames the possibly bad one. */
127 xdata_req = dict_new();
128 if (!xdata_req)
129 goto out;
130
131 pending = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
132 if (!pending)
133 goto out;
134
135 ret = afr_set_pending_dict(priv, xdata_req, pending);
136 if (ret < 0)
137 goto out;
138
139 if (local->fd) {
140 ret = syncop_fxattrop(priv->children[query_child], local->fd,
141 GF_XATTROP_ADD_ARRAY, xdata_req, NULL, &xdata_rsp,
142 NULL);
143 } else {
144 ret = syncop_xattrop(priv->children[query_child], &local->loc,
145 GF_XATTROP_ADD_ARRAY, xdata_req, NULL, &xdata_rsp,
146 NULL);
147 }
148 if (ret || !xdata_rsp) {
149 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
150 "Failed xattrop for gfid %s on %s",
151 uuid_utoa(local->inode->gfid),
152 priv->children[query_child]->name);
153 op_errno = -ret;
154 goto out;
155 }
156
157 if (afr_ta_dict_contains_pending_xattr(xdata_rsp, priv,
158 possible_bad_child)) {
159 read_subvol = query_child;
160 goto out;
161 }
162 dict_unref(xdata_rsp);
163 xdata_rsp = NULL;
164
165 /* It doesn't. So query thin-arbiter to see if it blames any data brick. */
166 ret = afr_fill_ta_loc(this, &loc, _gf_true);
167 if (ret) {
168 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
169 "Failed to populate thin-arbiter loc for: %s.", loc.name);
170 goto out;
171 }
172 flock.l_type = F_WRLCK; /*start and length are already zero. */
173 ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
174 AFR_TA_DOM_MODIFY, &loc, F_SETLKW, &flock, NULL, NULL);
175 if (ret) {
176 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
177 "gfid:%s: Failed to get AFR_TA_DOM_MODIFY lock on %s.",
178 uuid_utoa(local->inode->gfid),
179 priv->pending_key[THIN_ARBITER_BRICK_INDEX]);
180 op_errno = -ret;
181 goto out;
182 }
183
184 ret = syncop_xattrop(priv->children[THIN_ARBITER_BRICK_INDEX], &loc,
185 GF_XATTROP_ADD_ARRAY, xdata_req, NULL, &xdata_rsp,
186 NULL);
187 if (ret || !xdata_rsp) {
188 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
189 "gfid:%s: Failed xattrop on %s.", uuid_utoa(local->inode->gfid),
190 priv->pending_key[THIN_ARBITER_BRICK_INDEX]);
191 op_errno = -ret;
192 goto unlock;
193 }
194
195 if (!afr_ta_dict_contains_pending_xattr(xdata_rsp, priv, query_child)) {
196 read_subvol = query_child;
197 } else {
198 gf_msg(this->name, GF_LOG_ERROR, EIO, AFR_MSG_THIN_ARB,
199 "Failing read for gfid %s since good brick %s is down",
200 uuid_utoa(local->inode->gfid),
201 priv->children[possible_bad_child]->name);
202 op_errno = EIO;
203 }
204
205 unlock:
206 flock.l_type = F_UNLCK;
207 ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
208 AFR_TA_DOM_MODIFY, &loc, F_SETLK, &flock, NULL, NULL);
209 if (ret) {
210 gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
211 "gfid:%s: Failed to unlock AFR_TA_DOM_MODIFY lock on "
212 "%s.",
213 uuid_utoa(local->inode->gfid),
214 priv->pending_key[THIN_ARBITER_BRICK_INDEX]);
215 }
216 out:
217 if (xdata_req)
218 dict_unref(xdata_req);
219 if (xdata_rsp)
220 dict_unref(xdata_rsp);
221 if (pending)
222 afr_matrix_cleanup(pending, priv->child_count);
223 loc_wipe(&loc);
224
225 if (read_subvol == -1) {
226 local->op_ret = -1;
227 local->op_errno = op_errno;
228 }
229 afr_read_txn_wind(frame, this, read_subvol);
230 return ret;
231 }
232
233 static void
234 afr_ta_read_txn_synctask(call_frame_t *frame, xlator_t *this)
235 {
236 call_frame_t *ta_frame = NULL;
237 afr_local_t *local = NULL;
238 int ret = 0;
239
240 local = frame->local;
241 ta_frame = afr_ta_frame_create(this);
242 if (!ta_frame) {
243 local->op_ret = -1;
244 local->op_errno = ENOMEM;
245 gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
246 "Failed to create ta_frame");
247 goto out;
248 }
249 ret = synctask_new(this->ctx->env, afr_ta_read_txn, afr_ta_read_txn_done,
250 ta_frame, frame);
251 if (ret) {
252 gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
253 "Failed to launch "
254 "afr_ta_read_txn synctask for gfid %s.",
255 uuid_utoa(local->inode->gfid));
256 local->op_ret = -1;
257 local->op_errno = ENOMEM;
258 STACK_DESTROY(ta_frame->root);
259 goto out;
260 }
261 return;
262 out:
263 afr_read_txn_wind(frame, this, -1);
264 }
265
266 static int
267 afr_read_txn_refresh_done(call_frame_t *frame, xlator_t *this, int err)
268 {
269 afr_private_t *priv = NULL;
270 afr_local_t *local = NULL;
271 int read_subvol = -1;
272 inode_t *inode = NULL;
273 int ret = -1;
274 int spb_subvol = -1;
275
276 local = frame->local;
277 inode = local->inode;
278 priv = this->private;
279
280 if (err) {
281 if (!priv->thin_arbiter_count)
282 goto readfn;
283 if (err != EINVAL)
284 goto readfn;
285 /* We need to query the good bricks and/or thin-arbiter.*/
286 afr_ta_read_txn_synctask(frame, this);
287 return 0;
288 }
289
290 read_subvol = afr_read_subvol_select_by_policy(inode, this, local->readable,
291 NULL);
292 if (read_subvol == -1) {
293 err = EIO;
294 goto readfn;
295 }
296
297 if (local->read_attempted[read_subvol]) {
298 afr_read_txn_next_subvol(frame, this);
299 return 0;
300 }
301
302 local->read_attempted[read_subvol] = 1;
303 readfn:
304 if (read_subvol == -1) {
305 ret = afr_split_brain_read_subvol_get(inode, this, frame, &spb_subvol);
306 if ((ret == 0) && spb_subvol >= 0)
307 read_subvol = spb_subvol;
308 }
309
310 if (read_subvol == -1) {
311 AFR_SET_ERROR_AND_CHECK_SPLIT_BRAIN(-1, err);
312 }
313 afr_read_txn_wind(frame, this, read_subvol);
314
315 return 0;
316 }
317
318 int
319 afr_read_txn_continue(call_frame_t *frame, xlator_t *this, int subvol)
320 {
321 afr_local_t *local = NULL;
322
323 local = frame->local;
324
325 if (!local->refreshed) {
326 local->refreshed = _gf_true;
327 afr_inode_refresh(frame, this, local->inode, NULL,
328 afr_read_txn_refresh_done);
329 } else {
330 afr_read_txn_next_subvol(frame, this);
331 }
332
333 return 0;
334 }
335
336 /* afr_read_txn_wipe:
337
338 clean internal variables in @local in order to make
339 it possible to call afr_read_txn() multiple times from
340 the same frame
341 */
342
343 static void
344 afr_read_txn_wipe(call_frame_t *frame, unsigned int child_count)
345 {
346 afr_local_t *local = NULL;
347 int i = 0;
348
349 local = frame->local;
350
351 local->readfn = NULL;
352
353 if (local->inode)
354 inode_unref(local->inode);
355
356 for (i = 0; i < child_count; i++) {
357 local->read_attempted[i] = 0;
358 local->readable[i] = 0;
359 }
360 }
361
362 /*
363 afr_read_txn:
364
365 This is the read transaction function. The way it works:
366
367 - Determine read-subvolume from inode ctx.
368
369 - If read-subvolume's generation was stale, refresh ctx once by
370 calling afr_inode_refresh()
371
372 Else make an attempt to read on read-subvolume.
373
374 - If attempted read on read-subvolume fails, refresh ctx once
375 by calling afr_inode_refresh()
376
377 - After ctx refresh, query read-subvolume freshly and attempt
378 read once.
379
380 - If read fails, try every other readable[] subvolume before
381 finally giving up. readable[] elements are set by afr_inode_refresh()
382 based on dirty and pending flags.
383
384 - If file is in split brain in the backend, generation will be
385 kept 0 by afr_inode_refresh() and readable[] will be set 0 for
386 all elements. Therefore reads always fail.
387 */
388
389 int
390 afr_read_txn(call_frame_t *frame, xlator_t *this, inode_t *inode,
391 afr_read_txn_wind_t readfn, afr_transaction_type type)
392 {
393 afr_local_t *local = NULL;
394 afr_private_t *priv = NULL;
395 unsigned char *data = NULL;
396 unsigned char *metadata = NULL;
397 int read_subvol = -1;
398 int event_generation = 0;
399 int ret = -1;
400
401 priv = this->private;
402 local = frame->local;
403 data = alloca0(priv->child_count);
404 metadata = alloca0(priv->child_count);
405
406 afr_read_txn_wipe(frame, priv->child_count);
407
408 local->readfn = readfn;
409 local->inode = inode_ref(inode);
410 local->is_read_txn = _gf_true;
411 local->transaction.type = type;
412
413 if (priv->quorum_count && !afr_has_quorum(local->child_up, priv, NULL)) {
414 local->op_ret = -1;
415 local->op_errno = afr_quorum_errno(priv);
416 goto read;
417 }
418
419 if (!afr_is_consistent_io_possible(local, priv, &local->op_errno)) {
420 local->op_ret = -1;
421 goto read;
422 }
423
424 if (priv->thin_arbiter_count && !afr_ta_has_quorum(priv, local)) {
425 local->op_ret = -1;
426 local->op_errno = -afr_quorum_errno(priv);
427 goto read;
428 }
429
430 if (priv->thin_arbiter_count &&
431 AFR_COUNT(local->child_up, priv->child_count) != priv->child_count) {
432 if (local->child_up[0]) {
433 local->read_txn_query_child = AFR_CHILD_ZERO;
434 } else if (local->child_up[1]) {
435 local->read_txn_query_child = AFR_CHILD_ONE;
436 }
437 afr_ta_read_txn_synctask(frame, this);
438 return 0;
439 }
440
441 ret = afr_inode_read_subvol_get(inode, this, data, metadata,
442 &event_generation);
443 if (ret == -1)
444 /* very first transaction on this inode */
445 goto refresh;
446 AFR_INTERSECT(local->readable, data, metadata, priv->child_count);
447
448 gf_msg_debug(this->name, 0,
449 "%s: generation now vs cached: %d, "
450 "%d",
451 uuid_utoa(inode->gfid), local->event_generation,
452 event_generation);
453 if (afr_is_inode_refresh_reqd(inode, this, local->event_generation,
454 event_generation))
455 /* servers have disconnected / reconnected, and possibly
456 rebooted, very likely changing the state of freshness
457 of copies */
458 goto refresh;
459
460 read_subvol = afr_read_subvol_select_by_policy(inode, this, local->readable,
461 NULL);
462
463 if (read_subvol < 0 || read_subvol > priv->child_count) {
464 gf_msg_debug(this->name, 0,
465 "Unreadable subvolume %d found "
466 "with event generation %d for gfid %s.",
467 read_subvol, event_generation, uuid_utoa(inode->gfid));
468 goto refresh;
469 }
470
471 if (!local->child_up[read_subvol]) {
472 /* should never happen, just in case */
473 gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_READ_SUBVOL_ERROR,
474 "subvolume %d is the "
475 "read subvolume in this generation, but is not up",
476 read_subvol);
477 goto refresh;
478 }
479
480 local->read_attempted[read_subvol] = 1;
481
482 read:
483 afr_read_txn_wind(frame, this, read_subvol);
484
485 return 0;
486
487 refresh:
488 afr_inode_refresh(frame, this, inode, NULL, afr_read_txn_refresh_done);
489
490 return 0;
491 }