Debian Codegraph @

subversion/tests/cmdline/svnfsfs_tests.py

raw
1 #!/usr/bin/env python
2 #
3 # svnfsfs_tests.py: testing the 'svnfsfs' tool.
4 #
5 # Subversion is a tool for revision control.
6 # See http://subversion.apache.org for more information.
7 #
8 # ====================================================================
9 # Licensed to the Apache Software Foundation (ASF) under one
10 # or more contributor license agreements. See the NOTICE file
11 # distributed with this work for additional information
12 # regarding copyright ownership. The ASF licenses this file
13 # to you under the Apache License, Version 2.0 (the
14 # "License"); you may not use this file except in compliance
15 # with the License. You may obtain a copy of the License at
16 #
17 # http://www.apache.org/licenses/LICENSE-2.0
18 #
19 # Unless required by applicable law or agreed to in writing,
20 # software distributed under the License is distributed on an
21 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
22 # KIND, either express or implied. See the License for the
23 # specific language governing permissions and limitations
24 # under the License.
25 ######################################################################
26
27 # General modules
28 import os
29 import logging
30 import re
31 import shutil
32 import sys
33 import threading
34 import time
35 import gzip
36
37 logger = logging.getLogger()
38
39 # Our testing module
40 import svntest
41 from svntest.verify import SVNExpectedStdout, SVNExpectedStderr
42 from svntest.verify import SVNUnexpectedStderr
43 from svntest.verify import UnorderedOutput
44 from svntest.main import SVN_PROP_MERGEINFO
45
46 # (abbreviation)
47 Skip = svntest.testcase.Skip_deco
48 SkipUnless = svntest.testcase.SkipUnless_deco
49 XFail = svntest.testcase.XFail_deco
50 Issues = svntest.testcase.Issues_deco
51 Issue = svntest.testcase.Issue_deco
52 Wimp = svntest.testcase.Wimp_deco
53 SkipDumpLoadCrossCheck = svntest.testcase.SkipDumpLoadCrossCheck_deco
54 Item = svntest.wc.StateItem
55
56 #----------------------------------------------------------------------
57
58 # How we currently test 'svnfsfs' --
59 #
60 # 'svnfsfs stats': Run this on a greek repo, then verify that the
61 # various sections are present. The section contents
62 # is matched against section-specific patterns.
63 #
64 # 'svnfsfs dump-index': Tested implicitly by the load-index test
65 #
66 # 'svnfsfs load-index': Create a greek repo but set shard to 2 and pack
67 # it so we can load into a packed shard with more
68 # than one revision to test ordering issues etc.
69 # r1 also contains a non-trival number of items such
70 # that parser issues etc. have a chance to surface.
71 #
72 # The idea is dump the index of the pack, mess with
73 # it to cover lots of UI guarantees but keep the
74 # semantics of the relevant bits. Then feed it back
75 # to load-index and verify that the result is still
76 # a complete, consistent etc. repo.
77 #
78 ######################################################################
79 # Helper routines
80
81 def patch_format(repo_dir, shard_size):
82 """Rewrite the format of the FSFS repository REPO_DIR so
83 that it would use sharding with SHARDS revisions per shard."""
84
85 format_path = os.path.join(repo_dir, "db", "format")
86 contents = open(format_path, 'rb').read()
87 processed_lines = []
88
89 for line in contents.split(b"\n"):
90 if line.startswith(b"layout "):
91 processed_lines.append(b"layout sharded %d" % shard_size)
92 else:
93 processed_lines.append(line)
94
95 new_contents = b"\n".join(processed_lines)
96 os.chmod(format_path, svntest.main.S_ALL_RW)
97 with open(format_path, 'wb') as f:
98 f.write(new_contents)
99
100 ######################################################################
101 # Tests
102
103 #----------------------------------------------------------------------
104
105 @SkipUnless(svntest.main.is_fs_type_fsfs)
106 def test_stats(sbox):
107 "stats output"
108
109 sbox.build(create_wc=False)
110
111 exit_code, output, errput = \
112 svntest.actions.run_and_verify_svnfsfs(None, [], 'stats', sbox.repo_dir)
113
114 # split output into sections
115 sections = { }
116
117 last_line = ''
118 section_name = ''
119 section_contents = []
120 for line in output:
121 line = line.rstrip()
122 if line != '':
123
124 # If the first character is not a space, then LINE is a section header
125 if line[0] == ' ':
126 section_contents.append(line)
127 else:
128
129 # Store previous section
130 if section_name != '':
131 sections[section_name] = section_contents
132
133 # Is the output formatted nicely?
134 if last_line != '':
135 logger.warn("Error: no empty line before section '" + line + "'")
136 raise svntest.Failure
137
138 # start new section
139 section_name = line
140 section_contents = []
141
142 last_line = line
143
144 sections[section_name] = section_contents
145
146 # verify that these sections exist
147 sections_to_find = ['Reading revisions',
148 'Global statistics:',
149 'Noderev statistics:',
150 'Representation statistics:',
151 'Directory representation statistics:',
152 'File representation statistics:',
153 'Directory property representation statistics:',
154 'File property representation statistics:',
155 'Largest representations:',
156 'Extensions by number of representations:',
157 'Extensions by size of changed files:',
158 'Extensions by size of representations:',
159 'Histogram of expanded node sizes:',
160 'Histogram of representation sizes:',
161 'Histogram of file sizes:',
162 'Histogram of file representation sizes:',
163 'Histogram of file property sizes:',
164 'Histogram of file property representation sizes:',
165 'Histogram of directory sizes:',
166 'Histogram of directory representation sizes:',
167 'Histogram of directory property sizes:',
168 'Histogram of directory property representation sizes:']
169 patterns_to_find = {
170 'Reading revisions' : [r'\s+ 0[ 0-9]*'],
171 'Global .*' : [r'.*\d+ bytes in .*\d+ revisions',
172 r'.*\d+ bytes in .*\d+ changes',
173 r'.*\d+ bytes in .*\d+ node revision records',
174 r'.*\d+ bytes in .*\d+ representations',
175 r'.*\d+ bytes expanded representation size',
176 r'.*\d+ bytes with rep-sharing off' ],
177 'Noderev .*' : [r'.*\d+ bytes in .*\d+ nodes total',
178 r'.*\d+ bytes in .*\d+ directory noderevs',
179 r'.*\d+ bytes in .*\d+ file noderevs' ],
180 'Representation .*' : [r'.*\d+ bytes in .*\d+ representations total',
181 r'.*\d+ bytes in .*\d+ directory representations',
182 r'.*\d+ bytes in .*\d+ file representations',
183 r'.*\d+ bytes in .*\d+ representations of added file nodes',
184 r'.*\d+ bytes in .*\d+ directory property representations',
185 r'.*\d+ bytes in .*\d+ file property representations',
186 r'.*\d+ average delta chain length',
187 r'.*\d+ bytes in header & footer overhead' ],
188 '.* representation statistics:' :
189 [r'.*\d+ bytes in .*\d+ reps',
190 r'.*\d+ bytes in .*\d+ shared reps',
191 r'.*\d+ bytes expanded size',
192 r'.*\d+ bytes expanded shared size',
193 r'.*\d+ bytes with rep-sharing off',
194 r'.*\d+ shared references',
195 r'.*\d+ average delta chain length'],
196 'Largest.*:' : [r'.*\d+ r\d+ */\S*'],
197 'Extensions by number .*:' :
198 [r'.*\d+ \( ?\d+%\) representations'],
199 'Extensions by size .*:' :
200 [r'.*\d+ \( ?\d+%\) bytes'],
201 'Histogram of .*:' : [r'.*\d+ \.\. < \d+.*\d+ \( ?\d+%\) bytes in *\d+ \( ?\d+%\) items']
202 }
203
204 # check that the output contains all sections
205 for section_name in sections_to_find:
206 if not section_name in sections.keys():
207 logger.warn("Error: section '" + section_name + "' not found")
208 raise svntest.Failure
209
210 # check section contents
211 for section_name in sections.keys():
212 patterns = []
213
214 # find the suitable patterns for the current section
215 for pattern in patterns_to_find.keys():
216 if re.match(pattern, section_name):
217 patterns = patterns_to_find[pattern]
218 break;
219
220 if len(patterns) == 0:
221 logger.warn("Error: unexpected section '" + section_name + "' found'")
222 logger.warn(sections[section_name])
223 raise svntest.Failure
224
225 # each line in the section must match one of the patterns
226 for line in sections[section_name]:
227 found = False
228
229 for pattern in patterns:
230 if re.match(pattern, line):
231 found = True
232 break
233
234 if not found:
235 logger.warn("Error: unexpected line '" + line + "' in section '"
236 + section_name + "'")
237 logger.warn(sections[section_name])
238 raise svntest.Failure
239
240 #----------------------------------------------------------------------
241
242 @SkipUnless(svntest.main.is_fs_type_fsfs)
243 @SkipUnless(svntest.main.fs_has_pack)
244 @SkipUnless(svntest.main.is_fs_log_addressing)
245 def load_index_sharded(sbox):
246 "load-index in a packed repo"
247
248 # Configure two files per shard to trigger packing.
249 sbox.build(create_wc=False)
250 patch_format(sbox.repo_dir, shard_size=2)
251
252 expected_output = ["Packing revisions in shard 0...done.\n"]
253 svntest.actions.run_and_verify_svnadmin(expected_output, [],
254 "pack", sbox.repo_dir)
255
256 # Read P2L index using svnfsfs.
257 exit_code, items, errput = \
258 svntest.actions.run_and_verify_svnfsfs(None, [], "dump-index", "-r0",
259 sbox.repo_dir)
260
261 # load-index promises to deal with input that
262 #
263 # * uses the same encoding as the dump-index output
264 # * is not in ascending item offset order
265 # * contains lines with the full table header
266 # * invalid or incorrect data in the checksum column and beyond
267 # * starts with an item which does not belong to the first revision
268 # in the pack file
269 #
270 # So, let's mess with the ITEMS list to call in on these promises.
271
272 # not in ascending order
273 items.reverse()
274
275 # multiple headers (there is already one now at the bottom)
276 items.insert(0, " Start Length Type Revision Item Checksum\n")
277
278 # make columns have a variable size
279 # mess with the checksums
280 # add a junk column
281 # keep header lines as are
282 for i in range(0, len(items)):
283 if items[i].find("Start") == -1:
284 columns = items[i].split()
285 columns[5] = columns[5].replace('f','-').replace('0','9')
286 columns.append("junk")
287 items[i] = ' '.join(columns) + "\n"
288
289 # first entry shall be for rev 1, pack starts at rev 0, though
290 for i in range(0, len(items)):
291 if items[i].split()[3] == "1":
292 if i != 1:
293 items[i],items[1] = items[1],items[i]
294 break
295
296 assert(items[1].split()[3] == "1")
297
298 # The STDIN data must be binary.
299 items = svntest.main.ensure_list(map(str.encode, items))
300
301 # Reload the index
302 exit_code, output, errput = svntest.main.run_command_stdin(
303 svntest.main.svnfsfs_binary, [], 0, False, items,
304 "load-index", sbox.repo_dir)
305
306 # Run verify to see whether we broke anything.
307 expected_output = ["* Verifying metadata at revision 0 ...\n",
308 "* Verifying repository metadata ...\n",
309 "* Verified revision 0.\n",
310 "* Verified revision 1.\n"]
311 svntest.actions.run_and_verify_svnadmin(expected_output, [],
312 "verify", sbox.repo_dir)
313
314 @SkipUnless(svntest.main.is_fs_type_fsfs)
315 def test_stats_on_empty_repo(sbox):
316 "stats on empty repo shall not crash"
317
318 sbox.build(create_wc=False, empty=True)
319
320 exit_code, output, errput = \
321 svntest.actions.run_and_verify_svnfsfs(None, [], 'stats', sbox.repo_dir)
322
323 ########################################################################
324 # Run the tests
325
326
327 # list all tests here, starting with None:
328 test_list = [ None,
329 test_stats,
330 load_index_sharded,
331 test_stats_on_empty_repo,
332 ]
333
334 if __name__ == '__main__':
335 svntest.main.run_tests(test_list)
336 # NOTREACHED
337
338
339 ### End of file.