Debian Codegraph @

bin/lift

raw
1 #!/usr/bin/python3
2
3 # LIFT Integration-Functional Testing - A meta test framework
4 # Copyright © 2014-2021 Cognacq-Jay Image and Nicolas Delvaux
5 #
6 # This program is free software; you can redistribute it and/or
7 # modify it under the terms of the GNU General Public License
8 # as published by the Free Software Foundation; either version 2
9 # of the License, or (at your option) any later version.
10 #
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
15 #
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
19 # USA.
20
21 """The lift binary"""
22
23 import os
24 import re
25 import sys
26 import argparse
27
28 from junit_xml import TestSuite
29
30 import lift
31 from lift.exception import InvalidDescriptionFile
32 from lift.loader import load_config_file, load_upper_inheritance, string_to_remote
33
34
35 def parse():
36 """Command line argument parsing"""
37
38 def is_dir(parser, path):
39 """Additional type checker for argparse, validate a directory path"""
40 if not os.path.isdir(path):
41 parser.error(f"{path}: no such directory.")
42 else:
43 return path
44
45 def is_remote(parser, rstring):
46 """Additional type checker for argparse, validate a remote string"""
47 res = string_to_remote(rstring)
48 if res is None:
49 parser.error(f"{rstring} is not a valid remote string.")
50 else:
51 return res
52
53 parser = argparse.ArgumentParser()
54 parser.add_argument("-V", "--version", action="version", version=lift.version)
55 parser.add_argument(
56 "-n", "--no-color", action="store_true", help="Disable colored output"
57 )
58 parser.add_argument(
59 "-q",
60 "--quiet",
61 action="store_true",
62 help="Do not print the output of tests as they run",
63 )
64 parser.add_argument(
65 "-d",
66 "--detailed-summary",
67 action="store_true",
68 help="Print the output of failed test in the final" " summary",
69 )
70 parser.add_argument(
71 "--regex",
72 action="store_true",
73 help="Process test_expression as standard Python regex",
74 )
75 parser.add_argument(
76 "--no-upper-inheritance",
77 action="store_true",
78 help="Do not load remotes/environment from upper level" " lift.yaml files",
79 )
80 parser.add_argument(
81 "-f",
82 "--folder",
83 type=lambda p: is_dir(parser, p),
84 default=".",
85 help="Specify the root folder in which tests will be "
86 "looked for (default to the current working directory)",
87 )
88 parser.add_argument(
89 "-r",
90 "--remote",
91 type=lambda p: is_remote(parser, p),
92 action="append",
93 help="Define a remote. The value should be in the "
94 'following form: "REMOTENAME=USERNAME:PASSWORD@HOST". '
95 'Note that the PASSWORD field (along with the ":" '
96 "separator) is optional if SSH keys are properly set. "
97 "This option can be used multiple times to define "
98 "multiple remotes. Remotes defined via this option "
99 "supersede those defined via lift.yaml files.",
100 )
101 parser.add_argument(
102 "--put-remotes-in-environment",
103 action="store_true",
104 help="All defined remotes will be passed as "
105 "environment variables to tests. Variables will be "
106 "in the following form: "
107 '"LIFT_REMOTE_remotename=login:password@host". '
108 'Note that the password (along with the ":" '
109 "separator) will not be there if it was not defined "
110 "in the first place. Please do not use this option "
111 "if some of your binary tests can not be trusted to "
112 "keep these credentials for themselves.",
113 )
114 parser.add_argument(
115 "--with-xunit",
116 action="store_true",
117 help="Provide test results in the standard XUnit XML " "format.",
118 )
119 parser.add_argument(
120 "--xunit-file",
121 default="./lift.xml",
122 help="Path of the xml file to store the XUnit report "
123 "in. Default is lift.xml in the working directory.",
124 )
125 parser.add_argument(
126 "test_expression",
127 nargs="*",
128 help="Specify tests to run. By default, all discovered"
129 " tests are ran. "
130 "You can use the same format as the one found in the"
131 ' lift output, ie. "FOLDER/TEST_NAME". '
132 'If you set the "--regex" option, expressions will be'
133 ' matched as standard Python regex. For example, ".*foo"'
134 ' will match all tests containing the word "foo". '
135 "See http://docs.python.org/library/re.html for more "
136 "information.",
137 )
138 return parser.parse_args()
139
140
141 if __name__ != "__main__":
142 sys.exit("The lift binary can only be executed.")
143
144
145 # Parse arguments
146 args = parse()
147
148 if args.remote is not None:
149 preset_remotes = dict(args.remote)
150 else:
151 preset_remotes = {}
152
153 # Do we have a description file to parse?
154 if not os.path.isfile(os.path.join(args.folder, "lift.yaml")):
155 sys.exit("No lift.yaml file found in this folder.")
156
157 # Folder mapping, used for remotes and environment inheritance
158 mapping = []
159
160 # Load remotes/environment from upper level lift.yaml files
161 if not args.no_upper_inheritance:
162 remotes, environment = load_upper_inheritance(args.folder, preset_remotes)
163 else:
164 remotes = {}
165 environment = {}
166
167 # Initialize variables needed for the final summary
168 tests_count = 0.0
169 all_failed_tests = []
170 test_suites = []
171
172 for directory, _, _ in os.walk(args.folder):
173 if not os.path.isfile(os.path.join(directory, "lift.yaml")):
174 continue
175
176 # Figures out the inheritance of remotes and environments
177
178 # inherit from the closer known upper folder
179 for ancestor in reversed(mapping):
180 if not directory.startswith(ancestor["dir"]):
181 continue
182 remotes = ancestor["remotes"].copy()
183 environment = ancestor["environment"].copy()
184 break
185
186 # Load the description file
187 try:
188 tests, remotes, environment = load_config_file(
189 os.path.join(directory, "lift.yaml"),
190 remotes,
191 environment,
192 preset_remotes,
193 args.put_remotes_in_environment,
194 )
195 except InvalidDescriptionFile as e:
196 sys.exit(
197 f"{os.path.join(directory, 'lift.yaml')} is not a valid description file: {e}"
198 )
199
200 mapping.append({"dir": directory, "remotes": remotes, "environment": environment})
201 test_suites.append(TestSuite(directory, tests))
202
203 for test in tests:
204 test_string = f"{directory}/{test.name}"
205
206 # Should we ignore this test?
207 if args.test_expression:
208 if not args.regex and test_string not in args.test_expression:
209 # This is not one of the test the user explicitly asked for
210 test.add_skipped_info("Not selected from the command line.")
211 continue
212 if args.regex:
213 matched = False
214 for regex in args.test_expression:
215 if re.match(regex, test_string):
216 matched = True
217 break
218 if not matched:
219 test.add_skipped_info("Not selected from the command line.")
220 # test not matched, ignore it
221 continue
222
223 tests_count += 1
224 print("\nTesting: {0:-<{1}}".format(test_string + " ", 71))
225 if not args.quiet:
226 test.streaming_output = sys.stdout
227
228 cwd = os.getcwd()
229 status = test.run()
230 os.chdir(cwd) # Tests may change directory and fail to cleanup
231
232 res_string = "\n%s {0:>{1}}\n" % test_string
233 if status:
234 # TODO: align status according to output size
235 if not args.no_color:
236 # Green
237 print("\nResult: \033[92mOK\033[0m")
238 else:
239 print("\nResult: OK")
240 else:
241 all_failed_tests.append(test)
242 if not args.no_color:
243 # Red
244 print("\nResult: \033[91mFAIL\033[0m")
245 else:
246 print("\nResult: FAIL")
247
248 # All tests were run, summary time
249 if tests_count == 0:
250 sys.exit("No test was run!")
251
252 print("\nEnd of tests.")
253 if all_failed_tests:
254 print("=" * 80)
255 print("\nSummary of failed tests:\n")
256 for test in all_failed_tests:
257 if test.return_code != test.expected_return_code:
258 print(
259 f"\n{test.directory}/{test.name} returned {test.return_code} instead of {test.expected_return_code}\n"
260 )
261 if args.detailed_summary and test.output:
262 print(f"The output was:\n{test.output}\n")
263
264 print("####")
265
266 success_count = tests_count - len(all_failed_tests)
267 print(
268 "\nPass rate: %d/%d (%d%%)\n"
269 % (success_count, tests_count, int(round((success_count / tests_count) * 100)))
270 )
271
272 if args.with_xunit:
273 with open(args.xunit_file, "w") as f:
274 TestSuite.to_file(f, test_suites, prettyprint=False)
275
276 if all_failed_tests:
277 sys.exit(1)
278 else:
279 print("Congratulation! 👍\n")