1#!/usr/bin/env python3

2# -*- coding: utf-8 -*-

3#

4# This file is part of Checkbox.

5#

6# Copyright 2014 Canonical Ltd.

7#

8# Authors

9# Jeff Lane <jeffrey.lane@canonical.com>

10# Daniel Manrique<daniel.manrique@canonical.com>

11#

12# Checkbox is free software: you can redistribute it and/or modify

13# it under the terms of the GNU General Public License version 3,

14# as published by the Free Software Foundation.

15#

16# Checkbox is distributed in the hope that it will be useful,

17# but WITHOUT ANY WARRANTY; without even the implied warranty of

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

19# GNU General Public License for more details.

20#

21# You should have received a copy of the GNU General Public License

22# along with Checkbox. If not, see <http://www.gnu.org/licenses/>.

23

24

25"""

26 This script is used to parse the log generated by fwts and check it for certain

27 errors detected during testing. It expects that this is a log file created by

28 fwts at runtime using the -l <log name> option.

29

30 It's written now specifically for checking ater the fwts s3 and s4 tests but

31 can be adapted to look for other tests, or all tests.

32"""

33

34 import sys

35 import collections

36 import re

37

38 from argparse import ArgumentParser

39 import logging

40

41# Definitions of when a level starts, how a failure looks,

42# and when a level ends.

43 start_level_re = r"^(?P<level>.+) failures: (?P<numfails>NONE|\d+)$"

44 start_level_re = re.compile(start_level_re)

45 failure_re = re.compile(r"^ (?P<test>(\w*)): (?P<details>.+)$")

46 end_level_re = re.compile(r"$^")

47

48

49 def parse_summary(summary, results, filter_test, ignore_warning):

50 """

51 Parses an entire "Test Failure Summary" section, which contains a short

52 summary of failures observed per level. Returns nothing, but adds the

53 results to the passed results dictionary.

54

55 :param summary:

56 A list of lines comprised in this summary section.

57 :param results:

58 The results dictionary into which to put the end result. Should be a

59 dict with keys for each level, the values are dicts with keys for each

60 test (s3, s4) which in turn contain a list of all the failures observed

61 for that level and test.

62 :param filter_test:

63 A string to filter out the results is match with the test type.

64 And `all` will not filter anything.

65 :param ignore_warning:

66 A bool if warning message need to be ignored.

67 :return:

68 None

69 """

70 current_level = None

71 current_acum = []

72

73 for logline in summary:

74 level_matches = start_level_re.search(logline)

75 if level_matches:

76 logging.debug("Found a level: %s", level_matches.group("level"))

77 current_level = level_matches.group("level")

78 elif end_level_re.search(logline) and current_level:

79 if current_level:

80 logging.debug(

81 "Current level (%s) has %s", current_level, current_acum

82 )

83 # By passing results[current_level] a key in results will be

84 # created for every level we see, regardless of whether it

85 # reports failures or not. This is OK because we can later

86 # check results' keys to ensure we saw at least one level; if

87 # results has no keys, it could mean a malformed fwts log file.

88 parse_level(

89 current_acum,

90 results[current_level],

91 filter_test,

92 ignore_warning,

93 )

94 else:

95 logging.debug("Discarding junk")

96 current_acum = []

97 current_level = None

98 else:

99 current_acum.append(logline)

100

101

102 def parse_level(

103 level_lines, level_results, level_filter_test, level_ignore_warning

104):

105 """

106 Parses the level's lines, appending the failures to the level's results.

107 level_results is a dictionary with a key per test type (s3, s4, and so on).

108 Returns nothing, but adds the results to the passed results dictionary for

109 this level.

110

111 :param level_lines:

112 A list of lines comprised in this level's list of failures.

113

114 :param level_results:

115 A dictionary containing this level's results. Should be a dict with

116 keys for each test, to which the failures for the level will be

117 appended.

118 :param level_filter_test:

119 A string to filter out the results is match with the test type.

120 And `all` will not filter anything.

121 :param level_ignore_warning:

122 A bool if warning message need to be ignored.

123 :return:

124 None

125 """

126 for failureline in level_lines:

127 failure_matches = failure_re.search(failureline)

128 if failure_matches:

129 test = failure_matches.group("test")

130 details = failure_matches.group("details")

131 logging.debug("fail %s was %s", test, details)

132 if level_filter_test == "all" or level_filter_test in test:

133 if level_ignore_warning:

134 if "Warning:" not in details:

135 level_results[test].append(details)

136 else:

137 level_results[test].append(details)

138

139

140 def main():

141 parser = ArgumentParser()

142 parser.add_argument(

143 "-d",

144 "--debug",

145 action="store_const",

146 const=logging.DEBUG,

147 default=logging.INFO,

148 help="Show debugging information.",

149 )

150 parser.add_argument(

151 "-v",

152 "--verbose",

153 action="store_true",

154 default=False,

155 help="Display each error discovered. May provide \

156 very long output. Also, this option will only \

157 provide a list of UNIQUE errors encountered in \

158 the log file. It will not display duplicates. \

159 Default is [%(default)s]",

160 )

161 parser.add_argument(

162 "-t",

163 "--test",

164 action="store",

165 default="all",

166 help="The test (ie.all, s3 ,s4 ,klog ,oops ,...etc) \

167 to check. Default is [%(default)s]. And `all` \

168 is a special entry to check everything at once.",

169 )

170 parser.add_argument(

171 "logfile", action="store", help="The log file to parse"

172 )

173 parser.add_argument(

174 "--ignore-warning",

175 action="store_true",

176 default=False,

177 help="If ignore warning option is set, it will \

178 not show any warning items. \

179 Default is [%(default)s]",

180 )

181

182 args = parser.parse_args()

183

184 logging.basicConfig(level=args.debug)

185

186 # Create a generator and get our lines

187 log = (

188 line.rstrip() for line in open(args.logfile, "rt", encoding="UTF-8")

189 )

190

191 # End result will be a dictionary with a key per level, value is another

192 # dictionary with a key per test (s3, s4, ...) and a list of all failures

193 # for each test. Duplicates are possible, because we should also indicate

194 # the number of instances for each failure.

195 results = collections.defaultdict(lambda: collections.defaultdict(list))

196

197 sum_acum = []

198 summaries_found = 0

199

200 # Start parsing the fwts log. Gather each "Test Failure Summary" section

201 # and when it's complete, pass it to the parse_summary function to extract

202 # levels and tests.

203 for logline in log:

204 if "Test Failure Summary" in logline:

205 parse_summary(sum_acum, results, args.test, args.ignore_warning)

206 summaries_found += 1

207 sum_acum = []

208 else:

209 sum_acum.append(logline)

210 # We reached the end, so add the last accumulated summary

211 if sum_acum:

212 parse_summary(sum_acum, results, args.test, args.ignore_warning)

213

214 # Report what I found

215 for level in sorted(results.keys()):

216 if results[level]: # Yes, we can have an empty level.

217 # We may have seen the levelheader but had it report no failures.

218 print("{} failures:".format(level))

219 for test in results[level].keys():

220 print(

221 " {}: {} failures".format(test, len(results[level][test]))

222 )

223 if args.verbose:

224 print("=" * 40)

225 counts = collections.Counter(results[level][test])

226 for failure in counts:

227 print(" {} (x {})".format(failure, counts[failure]))

228

229 # Decide on the outcome based on the collected information

230 if not summaries_found:

231 logging.error(

232 "No fwts test summaries found, " "possible malformed fwts log file"

233 )

234 return_code = 2

235 elif not results.keys():

236 # If it has no keys, means we didn't see any FWTS levels

237 logging.error(

238 "None of the summaries contained failure levels, "

239 "possible malformed fwts log file"

240 )

241 return_code = 2

242 elif any(results.values()): # If any of the results' levels has errors

243 return_code = 1

244 else:

245 print("No errors detected")

246 return_code = 0

247

248 return return_code

249

250

251 if __name__ == "__main__":

252 sys.exit(main())