Debian Codegraph @

xlators/cluster/afr/src/afr-self-heal-metadata.c

raw
1 /*
2 Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
3 This file is part of GlusterFS.
4
5 This file is licensed to you under your choice of the GNU Lesser
6 General Public License, version 3 or any later version (LGPLv3 or
7 later), or the GNU General Public License, version 2 (GPLv2), in all
8 cases as published by the Free Software Foundation.
9 */
10
11 #include "afr.h"
12 #include "afr-self-heal.h"
13 #include "protocol-common.h"
14 #include <glusterfs/events.h>
15
16 #define AFR_HEAL_ATTR (GF_SET_ATTR_UID | GF_SET_ATTR_GID | GF_SET_ATTR_MODE)
17
18 static gf_boolean_t
19 _afr_ignorable_key_match(dict_t *d, char *k, data_t *val, void *mdata)
20 {
21 return afr_is_xattr_ignorable(k);
22 }
23
24 static void
25 afr_delete_ignorable_xattrs(dict_t *xattr)
26 {
27 dict_foreach_match(xattr, _afr_ignorable_key_match, NULL,
28 dict_remove_foreach_fn, NULL);
29 }
30
31 static int
32 __afr_selfheal_metadata_do(call_frame_t *frame, xlator_t *this, inode_t *inode,
33 int source, unsigned char *healed_sinks,
34 struct afr_reply *locked_replies)
35 {
36 int ret = -1;
37 loc_t loc = {
38 0,
39 };
40 dict_t *xattr = NULL;
41 dict_t *old_xattr = NULL;
42 afr_private_t *priv = NULL;
43 int i = 0;
44
45 priv = this->private;
46
47 loc.inode = inode_ref(inode);
48 gf_uuid_copy(loc.gfid, inode->gfid);
49
50 gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
51 "performing metadata selfheal on %s", uuid_utoa(inode->gfid));
52
53 ret = syncop_getxattr(priv->children[source], &loc, &xattr, NULL, NULL,
54 NULL);
55 if (ret < 0) {
56 ret = -EIO;
57 goto out;
58 }
59
60 afr_delete_ignorable_xattrs(xattr);
61
62 for (i = 0; i < priv->child_count; i++) {
63 if (old_xattr) {
64 dict_unref(old_xattr);
65 old_xattr = NULL;
66 }
67
68 if (!healed_sinks[i])
69 continue;
70
71 ret = syncop_setattr(priv->children[i], &loc,
72 &locked_replies[source].poststat, AFR_HEAL_ATTR,
73 NULL, NULL, NULL, NULL);
74 if (ret)
75 healed_sinks[i] = 0;
76
77 ret = syncop_getxattr(priv->children[i], &loc, &old_xattr, 0, NULL,
78 NULL);
79 if (old_xattr) {
80 afr_delete_ignorable_xattrs(old_xattr);
81 ret = syncop_removexattr(priv->children[i], &loc, "", old_xattr,
82 NULL);
83 if (ret)
84 healed_sinks[i] = 0;
85 }
86
87 ret = syncop_setxattr(priv->children[i], &loc, xattr, 0, NULL, NULL);
88 if (ret)
89 healed_sinks[i] = 0;
90 }
91 ret = 0;
92
93 out:
94 loc_wipe(&loc);
95 if (xattr)
96 dict_unref(xattr);
97 if (old_xattr)
98 dict_unref(old_xattr);
99
100 return ret;
101 }
102
103 static uint64_t
104 mtime_ns(struct iatt *ia)
105 {
106 uint64_t ret;
107
108 ret = (((uint64_t)(ia->ia_mtime)) * 1000000000) +
109 (uint64_t)(ia->ia_mtime_nsec);
110
111 return ret;
112 }
113
114 /*
115 * When directory content is modified, [mc]time is updated. On
116 * Linux, the filesystem does it, while at least on NetBSD, the
117 * kernel file-system independent code does it. This means that
118 * when entries are added while bricks are down, the kernel sends
119 * a SETATTR [mc]time which will cause metadata split brain for
120 * the directory. In this case, clear the split brain by finding
121 * the source with the most recent modification date.
122 */
123 static int
124 afr_dirtime_splitbrain_source(call_frame_t *frame, xlator_t *this,
125 struct afr_reply *replies,
126 unsigned char *locked_on)
127 {
128 afr_private_t *priv = NULL;
129 int source = -1;
130 struct iatt source_ia;
131 struct iatt child_ia;
132 uint64_t mtime = 0;
133 int i;
134 int ret = -1;
135
136 priv = this->private;
137
138 for (i = 0; i < priv->child_count; i++) {
139 if (!locked_on[i])
140 continue;
141
142 if (!replies[i].valid)
143 continue;
144
145 if (replies[i].op_ret != 0)
146 continue;
147
148 if (mtime_ns(&replies[i].poststat) <= mtime)
149 continue;
150
151 mtime = mtime_ns(&replies[i].poststat);
152 source = i;
153 }
154
155 if (source == -1)
156 goto out;
157
158 source_ia = replies[source].poststat;
159 if (source_ia.ia_type != IA_IFDIR)
160 goto out;
161
162 for (i = 0; i < priv->child_count; i++) {
163 if (i == source)
164 continue;
165
166 if (!replies[i].valid)
167 continue;
168
169 if (replies[i].op_ret != 0)
170 continue;
171
172 child_ia = replies[i].poststat;
173
174 if (!IA_EQUAL(source_ia, child_ia, gfid) ||
175 !IA_EQUAL(source_ia, child_ia, type) ||
176 !IA_EQUAL(source_ia, child_ia, prot) ||
177 !IA_EQUAL(source_ia, child_ia, uid) ||
178 !IA_EQUAL(source_ia, child_ia, gid) ||
179 !afr_xattrs_are_equal(
180 replies[source].xdata, replies[i].xdata,
181 AFR_IS_ARBITER_BRICK(priv, i) ? _gf_true : _gf_false))
182 goto out;
183 }
184
185 /*
186 * Metadata split brain is just about [amc]time
187 * We return our source.
188 */
189 ret = source;
190 out:
191 return ret;
192 }
193
194 static int
195 __afr_selfheal_metadata_mark_pending_xattrs(call_frame_t *frame, xlator_t *this,
196 inode_t *inode,
197 struct afr_reply *replies,
198 unsigned char *sources)
199 {
200 int ret = 0;
201 int i = 0;
202 int m_idx = 0;
203 afr_private_t *priv = NULL;
204 int raw[AFR_NUM_CHANGE_LOGS] = {0};
205 dict_t *xattr = NULL;
206
207 priv = this->private;
208 m_idx = afr_index_for_transaction_type(AFR_METADATA_TRANSACTION);
209 raw[m_idx] = 1;
210
211 xattr = dict_new();
212 if (!xattr)
213 return -ENOMEM;
214
215 for (i = 0; i < priv->child_count; i++) {
216 if (sources[i])
217 continue;
218 ret = dict_set_static_bin(xattr, priv->pending_key[i], raw,
219 sizeof(int) * AFR_NUM_CHANGE_LOGS);
220 if (ret) {
221 ret = -1;
222 goto out;
223 }
224 }
225
226 for (i = 0; i < priv->child_count; i++) {
227 if (!sources[i])
228 continue;
229 ret = afr_selfheal_post_op(frame, this, inode, i, xattr, NULL);
230 if (ret < 0) {
231 gf_msg(this->name, GF_LOG_INFO, -ret, AFR_MSG_SELF_HEAL_INFO,
232 "Failed to set pending metadata xattr on child %d for %s", i,
233 uuid_utoa(inode->gfid));
234 goto out;
235 }
236 }
237
238 afr_replies_wipe(replies, priv->child_count);
239 ret = afr_selfheal_unlocked_discover(frame, inode, inode->gfid, replies);
240
241 out:
242 if (xattr)
243 dict_unref(xattr);
244 return ret;
245 }
246
247 /*
248 * Look for mismatching uid/gid or mode or user xattrs even if
249 * AFR xattrs don't say so, and pick one arbitrarily as winner. */
250
251 static int
252 __afr_selfheal_metadata_finalize_source(call_frame_t *frame, xlator_t *this,
253 inode_t *inode, unsigned char *sources,
254 unsigned char *sinks,
255 unsigned char *healed_sinks,
256 unsigned char *undid_pending,
257 unsigned char *locked_on,
258 struct afr_reply *replies)
259 {
260 int i = 0;
261 afr_private_t *priv = NULL;
262 struct iatt srcstat = {
263 0,
264 };
265 int source = -1;
266 int sources_count = 0;
267 int ret = 0;
268
269 priv = this->private;
270
271 sources_count = AFR_COUNT(sources, priv->child_count);
272
273 if ((AFR_CMP(locked_on, healed_sinks, priv->child_count) == 0) ||
274 !sources_count) {
275 source = afr_mark_split_brain_source_sinks(
276 frame, this, inode, sources, sinks, healed_sinks, locked_on,
277 replies, AFR_METADATA_TRANSACTION);
278 if (source >= 0) {
279 _afr_fav_child_reset_sink_xattrs(
280 frame, this, inode, source, healed_sinks, undid_pending,
281 AFR_METADATA_TRANSACTION, locked_on, replies);
282 goto out;
283 }
284
285 /* If this is a directory mtime/ctime only split brain
286 use the most recent */
287 source = afr_dirtime_splitbrain_source(frame, this, replies, locked_on);
288 if (source != -1) {
289 gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SPLIT_BRAIN,
290 "clear time "
291 "split brain on %s",
292 uuid_utoa(replies[source].poststat.ia_gfid));
293 sources[source] = 1;
294 healed_sinks[source] = 0;
295 goto out;
296 }
297
298 if (!priv->metadata_splitbrain_forced_heal) {
299 gf_event(EVENT_AFR_SPLIT_BRAIN,
300 "client-pid=%d;"
301 "subvol=%s;"
302 "type=metadata;file=%s",
303 this->ctx->cmd_args.client_pid, this->name,
304 uuid_utoa(inode->gfid));
305 return -EIO;
306 }
307
308 /* Metadata split brain, select one subvol
309 arbitrarily */
310 for (i = 0; i < priv->child_count; i++) {
311 if (locked_on[i] && healed_sinks[i]) {
312 sources[i] = 1;
313 healed_sinks[i] = 0;
314 break;
315 }
316 }
317 }
318
319 /* No split brain at this point. If we were called from
320 * afr_heal_splitbrain_file(), abort.*/
321 if (afr_dict_contains_heal_op(frame))
322 return -EIO;
323
324 source = afr_choose_source_by_policy(priv, sources,
325 AFR_METADATA_TRANSACTION);
326 srcstat = replies[source].poststat;
327
328 for (i = 0; i < priv->child_count; i++) {
329 if (!sources[i] || i == source)
330 continue;
331 if (!IA_EQUAL(srcstat, replies[i].poststat, type) ||
332 !IA_EQUAL(srcstat, replies[i].poststat, uid) ||
333 !IA_EQUAL(srcstat, replies[i].poststat, gid) ||
334 !IA_EQUAL(srcstat, replies[i].poststat, prot)) {
335 gf_msg_debug(this->name, 0,
336 "%s: iatt mismatch "
337 "for source(%d) vs (%d)",
338 uuid_utoa(replies[source].poststat.ia_gfid), source,
339 i);
340 sources[i] = 0;
341 healed_sinks[i] = 1;
342 }
343 }
344
345 for (i = 0; i < priv->child_count; i++) {
346 if (!sources[i] || i == source)
347 continue;
348 if (!afr_xattrs_are_equal(
349 replies[source].xdata, replies[i].xdata,
350 AFR_IS_ARBITER_BRICK(priv, i) ? _gf_true : _gf_false)) {
351 gf_msg_debug(this->name, 0,
352 "%s: xattr mismatch "
353 "for source(%d) vs (%d)",
354 uuid_utoa(replies[source].poststat.ia_gfid), source,
355 i);
356 sources[i] = 0;
357 healed_sinks[i] = 1;
358 }
359 }
360
361 if ((sources_count == priv->child_count) && (source > -1) &&
362 (AFR_COUNT(healed_sinks, priv->child_count) != 0)) {
363 ret = __afr_selfheal_metadata_mark_pending_xattrs(frame, this, inode,
364 replies, sources);
365 if (ret < 0)
366 return ret;
367 }
368 out:
369 afr_mark_active_sinks(this, sources, locked_on, healed_sinks);
370 return source;
371 }
372
373 int
374 __afr_selfheal_metadata_prepare(call_frame_t *frame, xlator_t *this,
375 inode_t *inode, unsigned char *locked_on,
376 unsigned char *sources, unsigned char *sinks,
377 unsigned char *healed_sinks,
378 unsigned char *undid_pending,
379 struct afr_reply *replies, unsigned char *pflag)
380 {
381 int ret = -1;
382 int source = -1;
383 afr_private_t *priv = NULL;
384 int i = 0;
385 uint64_t *witness = NULL;
386
387 priv = this->private;
388
389 ret = afr_selfheal_unlocked_discover(frame, inode, inode->gfid, replies);
390 if (ret)
391 return ret;
392
393 witness = alloca0(sizeof(*witness) * priv->child_count);
394 ret = afr_selfheal_find_direction(frame, this, replies,
395 AFR_METADATA_TRANSACTION, locked_on,
396 sources, sinks, witness, pflag);
397 if (ret)
398 return ret;
399
400 /* Initialize the healed_sinks[] array optimistically to
401 the intersection of to-be-healed (i.e sinks[]) and
402 the list of servers which are up (i.e locked_on[]).
403
404 As we encounter failures in the healing process, we
405 will unmark the respective servers in the healed_sinks[]
406 array.
407 */
408 AFR_INTERSECT(healed_sinks, sinks, locked_on, priv->child_count);
409
410 /* If any source has witness, pick first
411 * witness source and make everybody else sinks */
412 for (i = 0; i < priv->child_count; i++) {
413 if (sources[i] && witness[i]) {
414 source = i;
415 break;
416 }
417 }
418
419 if (source != -1) {
420 for (i = 0; i < priv->child_count; i++) {
421 if (i != source && sources[i]) {
422 sources[i] = 0;
423 healed_sinks[i] = 1;
424 }
425 }
426 }
427
428 source = __afr_selfheal_metadata_finalize_source(
429 frame, this, inode, sources, sinks, healed_sinks, undid_pending,
430 locked_on, replies);
431
432 if (source < 0)
433 return -EIO;
434
435 return source;
436 }
437
438 int
439 afr_selfheal_metadata(call_frame_t *frame, xlator_t *this, inode_t *inode)
440 {
441 afr_private_t *priv = NULL;
442 int ret = -1;
443 unsigned char *sources = NULL;
444 unsigned char *sinks = NULL;
445 unsigned char *data_lock = NULL;
446 unsigned char *healed_sinks = NULL;
447 unsigned char *undid_pending = NULL;
448 struct afr_reply *locked_replies = NULL;
449 gf_boolean_t did_sh = _gf_true;
450 int source = -1;
451
452 priv = this->private;
453
454 sources = alloca0(priv->child_count);
455 sinks = alloca0(priv->child_count);
456 healed_sinks = alloca0(priv->child_count);
457 undid_pending = alloca0(priv->child_count);
458 data_lock = alloca0(priv->child_count);
459
460 locked_replies = alloca0(sizeof(*locked_replies) * priv->child_count);
461
462 ret = afr_selfheal_tie_breaker_inodelk(frame, this, inode, this->name,
463 LLONG_MAX - 1, 0, data_lock);
464 {
465 if (ret < priv->child_count) {
466 ret = -ENOTCONN;
467 goto unlock;
468 }
469
470 ret = __afr_selfheal_metadata_prepare(
471 frame, this, inode, data_lock, sources, sinks, healed_sinks,
472 undid_pending, locked_replies, NULL);
473 if (ret < 0)
474 goto unlock;
475
476 source = ret;
477
478 if (AFR_COUNT(healed_sinks, priv->child_count) == 0) {
479 did_sh = _gf_false;
480 goto unlock;
481 }
482
483 ret = __afr_selfheal_metadata_do(frame, this, inode, source,
484 healed_sinks, locked_replies);
485 if (ret)
486 goto unlock;
487
488 afr_selfheal_restore_time(frame, this, inode, source, healed_sinks,
489 locked_replies);
490
491 ret = afr_selfheal_undo_pending(
492 frame, this, inode, sources, sinks, healed_sinks, undid_pending,
493 AFR_METADATA_TRANSACTION, locked_replies, data_lock);
494 }
495 unlock:
496 afr_selfheal_uninodelk(frame, this, inode, this->name, LLONG_MAX - 1, 0,
497 data_lock);
498
499 if (did_sh)
500 afr_log_selfheal(inode->gfid, this, ret, "metadata", source, sources,
501 healed_sinks);
502 else
503 ret = 1;
504
505 if (locked_replies)
506 afr_replies_wipe(locked_replies, priv->child_count);
507 return ret;
508 }
509
510 int
511 afr_selfheal_metadata_by_stbuf(xlator_t *this, struct iatt *stbuf)
512 {
513 inode_t *inode = NULL;
514 inode_t *link_inode = NULL;
515 call_frame_t *frame = NULL;
516 int ret = 0;
517
518 if (gf_uuid_is_null(stbuf->ia_gfid)) {
519 ret = -EINVAL;
520 goto out;
521 }
522
523 inode = inode_new(this->itable);
524 if (!inode) {
525 ret = -ENOMEM;
526 goto out;
527 }
528
529 link_inode = inode_link(inode, NULL, NULL, stbuf);
530 if (!link_inode) {
531 ret = -ENOMEM;
532 goto out;
533 }
534
535 frame = afr_frame_create(this, &ret);
536 if (!frame) {
537 ret = -ret;
538 goto out;
539 }
540
541 ret = afr_selfheal_metadata(frame, this, link_inode);
542 out:
543 if (inode)
544 inode_unref(inode);
545 if (link_inode)
546 inode_unref(link_inode);
547 if (frame)
548 AFR_STACK_DESTROY(frame);
549 return ret;
550 }