1#!/usr/bin/env python3

2

3# Copyright (C) 2020 Canonical Ltd.

4#

5# Authors

6# Rod Smith <rod.smith@canonical.com>

7#

8# This program is free software: you can redistribute it and/or modify

9# it under the terms of the GNU General Public License version 3,

10# as published by the Free Software Foundation.

11#

12# This program is distributed in the hope that it will be useful,

13# but WITHOUT ANY WARRANTY; without even the implied warranty of

14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

15# GNU General Public License for more details.

16#

17# You should have received a copy of the GNU General Public License

18# along with this program. If not, see <http://www.gnu.org/licenses/>.

19

20"""

21 Run CPU, memory, and disk stress tests using the stress-ng binary program.

22"""

23

24

25 from argparse import ArgumentParser, RawTextHelpFormatter

26 from subprocess import (

27 CalledProcessError,

28 check_output,

29 PIPE,

30 Popen,

31 run,

32 STDOUT,

33 TimeoutExpired,

34)

35 import os

36 import shlex

37 import shutil

38 import stat

39 import sys

40 import time

41 import uuid

42 import psutil

43 from checkbox_support.disk_support import Disk

44

45# Swap filename

46 my_swap = None

47

48

49 class StressNg:

50 """Interface with the external stress-ng binary."""

51

52 # Accepts test parameters, runs the test, and enables access to test

53 # results.

54

55 def __init__(

56 self,

57 stressors,

58 wrapper_timeout,

59 sng_timeout,

60 thread_count=0,

61 extra_options="",

62 ):

63

64 self.stressors = stressors

65 self.wrapper_timeout = wrapper_timeout

66 self.sng_timeout = sng_timeout

67 self.extra_options = extra_options

68 self.thread_count = thread_count

69 self.results = ""

70 self.returncode = 0

71

72 def run(self):

73 """Run a stress-ng test, storing results in self.results."""

74

75 stressor_list = "--" + " {} --".format(self.thread_count).join(

76 self.stressors

77 )

78 # LP:1983122 ensure the final stressor in the list is properly defined

79 stressor_list = stressor_list + " {}".format(self.thread_count)

80

81 command = "stress-ng --aggressive --verify --timeout {} {} {}".format(

82 self.sng_timeout, self.extra_options, stressor_list

83 )

84 time_str = time.strftime("%d %b %H:%M", time.gmtime())

85 if len(self.stressors) == 1:

86 print(

87 "{}: Running stress-ng {} stressor for {:.0f} seconds...".format( # noqa: E501

88 time_str, self.stressors[0], self.sng_timeout

89 )

90 )

91 else:

92 print(

93 "{}: Running multiple stress-ng stressors in "

94 "parallel for {:.0f}".format(time_str, self.sng_timeout)

95 )

96 print("seconds...")

97 try:

98 self.results = check_output(

99 shlex.split(command), timeout=self.wrapper_timeout

100 ).decode(encoding=sys.stdout.encoding)

101 except CalledProcessError as err:

102 print("** stress-ng exited with code {}".format(err.returncode))

103 self.results = err.stdout.decode(encoding="utf-8")

104 self.returncode = err.returncode

105 except TimeoutExpired:

106 print("** stress-ng timed out and was forcefully terminated")

107 self.results = ""

108 self.returncode = 1

109 except KeyboardInterrupt:

110 self.results = ""

111 print("** stress-ng test was terminated by SIGINT (Ctrl+C)!")

112 self.returncode = 1

113 except FileNotFoundError:

114 print("** stress-ng binary not found!")

115 self.results = ""

116 self.returncode = 1

117 return self.returncode

118

119

120# Define CPU-related functions...

121

122

123 def stress_cpu(args):

124 """Run stress-ng tests on CPUs."""

125

126 retval = 0

127 stressors = [

128 "af-alg",

129 "bsearch",

130 "context",

131 "cpu",

132 "crypt",

133 "hsearch",

134 "longjmp",

135 "lsearch",

136 "matrix",

137 "qsort",

138 "str",

139 "stream",

140 "tsearch",

141 "vecmath",

142 "wcs",

143 ]

144 # Add 10% to runtime; will forcefully terminate if stress-ng

145 # fails to return in that time.

146 end_time = 1.1 * args.base_time

147 print(

148 "Estimated total run time is {:.0f} minutes\n".format(

149 args.base_time / 60

150 )

151 )

152

153 test_object = StressNg(

154 stressors=stressors,

155 sng_timeout=args.base_time,

156 wrapper_timeout=end_time,

157 extra_options="--metrics-brief --tz --times",

158 )

159 retval = test_object.run()

160 print(test_object.results)

161 return retval

162

163

164# Define memory-related functions...

165

166

167 def num_numa_nodes():

168 """Return the number of NUMA nodes supported by the CPU."""

169

170 try:

171 return int(

172 run(["numactl", "--hardware"], stdout=PIPE).stdout.split()[1]

173 )

174 except (ValueError, OSError, IndexError):

175 return 1

176

177

178 def swap_space_ok(swap_size):

179 """Check available swap space."""

180 # If swap space is too small, add more. The minimum

181 # acceptable amount is defined as the GREATER of the amount specified

182 # by the command-line -s/--swap-space option OR the amount specified

183 # by the STRESS_NG_MIN_SWAP_SIZE environment variable. Both values are

184 # specified in gibibytes (GiB). If neither is specified, a value of 0

185 # (no swap required) is assumed.

186 # Returns:

187 # - True if OK (already or after adding more)

188 # - False if insufficient swap space

189

190 all_ok = True

191 global my_swap

192 min_swap_space = 0

193

194 swap_size = max(os.environ.get("STRESS_NG_MIN_SWAP_SPACE", 0), swap_size)

195 print("Minimum swap space is set to {} GiB".format(swap_size))

196 min_swap_space = swap_size * 1024**3

197 swap = psutil.swap_memory()

198 if swap.total < min_swap_space:

199 print(

200 "Swap space too small! Attempting to add more (this may take "

201 + "a while)...."

202 )

203 my_swap = "/swap-{}.img".format(uuid.uuid1())

204 # Create swap file 10KiB bigger than minimum because there's a 4KiB

205 # overhead in the file, so if it were exactly the minimum, it would

206 # still be too small....

207 try:

208 with open(my_swap, "w+b") as f:

209 # Swap file zeroed out and increased in size in 1KiB chunks to

210 # avoid problems with sparse files and creating temporary RAM

211 # use that potentially exceeds available RAM....

212 for i in range(int((min_swap_space + 10240) / 1024)):

213 f.write(b"\x00" * 1024)

214 f.flush()

215 except OSError:

216 print("Unable to create temporary swap file! Aborting test!")

217 try:

218 # In case the file was partially written but errored out

219 # (say, because of a lack of disk space)

220 os.remove(my_swap)

221 except FileNotFoundError:

222 # This exception will happen if the file doesn't exist at all

223 pass

224 all_ok = False

225 if all_ok:

226 os.chmod(my_swap, stat.S_IRUSR | stat.S_IWUSR)

227 run(["mkswap", my_swap])

228 run(["swapon", my_swap])

229 swap = psutil.swap_memory()

230 return swap.total >= min_swap_space

231

232

233 def stress_memory(args):

234 """Run stress-ng tests on memory."""

235

236 retval = 0

237 if not swap_space_ok(args.swap_size):

238 print(

239 "** Swap space unavailable! Please activate swap space "

240 + "and re-run this test!"

241 )

242 return 1

243

244 ram = psutil.virtual_memory()

245 total_mem_in_gb = ram.total / (1024**3)

246 vrt = args.base_time + total_mem_in_gb * args.time_per_gig

247 print("Total memory is {:.1f} GiB".format(total_mem_in_gb))

248 print(

249 "Constant run time is {} seconds per stressor".format(args.base_time)

250 )

251 print("Variable run time is {:.0f} seconds per stressor".format(vrt))

252 print("Number of NUMA nodes is {}".format(num_numa_nodes()))

253

254 # Constant-run-time stressors -- run them for the same length of time on

255 # all systems....

256 crt_stressors = [

257 "bsearch",

258 "context",

259 "hsearch",

260 "lsearch",

261 "matrix",

262 "memcpy",

263 "null",

264 "pipe",

265 "qsort",

266 "str",

267 "stream",

268 "tsearch",

269 "vm-rw",

270 "wcs",

271 "zero",

272 "mlock",

273 "mmapfork",

274 "mmapmany",

275 "mremap",

276 "shm-sysv",

277 "vm-splice",

278 ]

279

280 if num_numa_nodes() > 1:

281 crt_stressors.append("numa")

282

283 # Variable-run-time stressors -- run longer on systems with more RAM....

284 vrt_stressors = ["malloc", "mincore", "vm", "mmap"]

285 # Low-thread-count stressors -- throttle to >8 threads...

286 ltc_stressors = ["stack", "bigheap", "brk"]

287

288 est_runtime = (

289 len(crt_stressors) * args.base_time + len(vrt_stressors) * vrt

290 )

291 print(

292 "Estimated total run time is {:.0f} minutes\n".format(est_runtime / 60)

293 )

294 for stressor in crt_stressors:

295 test_object = StressNg(

296 stressors=stressor.split(),

297 sng_timeout=args.base_time,

298 wrapper_timeout=args.base_time * 2,

299 thread_count=0,

300 )

301 retval = retval | test_object.run()

302 print(test_object.results)

303 for stressor in vrt_stressors:

304 test_object = StressNg(

305 stressors=stressor.split(),

306 sng_timeout=vrt,

307 wrapper_timeout=vrt * 2,

308 thread_count=0,

309 )

310 retval = retval | test_object.run()

311 print(test_object.results)

312 for stressor in ltc_stressors:

313 test_object = StressNg(

314 stressors=stressor.split(),

315 sng_timeout=vrt,

316 wrapper_timeout=vrt * 2,

317 thread_count=8,

318 ) # throttle to 8 threads

319 retval = retval | test_object.run()

320 print(test_object.results)

321 if my_swap is not None and args.keep_swap is False:

322 print("Deleting temporary swap file....")

323 cmd = "swapoff {}".format(my_swap)

324 Popen(shlex.split(cmd), stderr=STDOUT, stdout=PIPE).communicate()[0]

325 os.remove(my_swap)

326 return retval

327

328

329 def stress_disk(args):

330 """Run stress-ng tests on disk."""

331

332 disk_stressors = [

333 "aio",

334 "aiol",

335 "chdir",

336 "chmod",

337 "chown",

338 "dentry",

339 "dir",

340 "fallocate",

341 "fiemap",

342 "filename",

343 "flock",

344 "fstat",

345 "hdd",

346 "ioprio",

347 "lease",

348 "locka",

349 "lockf",

350 "lockofd",

351 "madvise",

352 "mknod",

353 "msync",

354 "readahead",

355 "seal",

356 "seek",

357 "sync-file",

358 "xattr",

359 ]

360

361 retval = 0

362 if "/dev" not in args.device and args.device != "":

363 args.device = "/dev/" + args.device

364

365 test_disk = Disk(args.device)

366 if not test_disk.is_block_device():

367 print("** {} is not a block device! Aborting!".format(args.device))

368 return 1

369 if test_disk.mount_filesystem(args.simulate):

370 est_runtime = len(disk_stressors) * args.base_time

371 print("Using test directory: '{}'".format(test_disk.test_dir))

372 print(

373 "Estimated total run time is {:.0f} minutes\n".format(

374 est_runtime / 60

375 )

376 )

377 retval = 0

378 if not args.simulate:

379 for stressor in disk_stressors:

380 disk_options = (

381 "--temp-path {} ".format(test_disk.test_dir)

382 + "--hdd-opts dsync --readahead-bytes 16M -k"

383 )

384 test_object = StressNg(

385 stressors=stressor.split(),

386 sng_timeout=args.base_time,

387 wrapper_timeout=args.base_time * 5,

388 extra_options=disk_options,

389 )

390 retval = retval | test_object.run()

391 print(test_object.results)

392 if test_disk.test_dir != "/tmp" and not args.simulate:

393 shutil.rmtree(test_disk.test_dir, ignore_errors=True)

394 else:

395 print("** Unable to find a suitable partition! Aborting!")

396 retval = 1

397

398 return retval

399

400

401# Main program body...

402

403

404 def main():

405 """Run a stress_ng-based stress run."""

406

407 parser = ArgumentParser(

408 description="Run tests based on stress-ng",

409 formatter_class=RawTextHelpFormatter,

410 )

411 subparsers = parser.add_subparsers()

412

413 # Main cli options

414 cpu_parser = subparsers.add_parser("cpu", help=("Run CPU tests"))

415 memory_parser = subparsers.add_parser("memory", help=("Run memory tests"))

416 disk_parser = subparsers.add_parser("disk", help=("Run disk tests"))

417

418 # CPU parameters

419 cpu_parser.add_argument(

420 "-b",

421 "--base-time",

422 type=int,

423 default=7200,

424 help="Run time, in seconds (default=7200)",

425 )

426

427 # Memory parameters

428 memory_parser.add_argument(

429 "-b",

430 "--base-time",

431 type=int,

432 help="Base time for each test, in seconds " + "(default=300)",

433 default=300,

434 )

435 memory_parser.add_argument(

436 "-t",

437 "--time-per-gig",

438 type=int,

439 help="Extra time per GiB for some stressors,"

440 + " in seconds (default=10)",

441 default=10,

442 )

443 memory_parser.add_argument(

444 "-s", "--swap-size", type=int, help="swap size in GiB", default=0

445 )

446 memory_parser.add_argument(

447 "-k",

448 "--keep-swap",

449 action="store_true",

450 help="Keep swap file, if added by test",

451 )

452

453 # Disk parameters

454 disk_parser.add_argument(

455 "-d",

456 "--device",

457 type=str,

458 required=True,

459 help="Disk device (/dev/sda, etc.)",

460 )

461 disk_parser.add_argument(

462 "-b",

463 "--base-time",

464 type=int,

465 help="Time for each test, in seconds " + "(default=240)",

466 default=240,

467 )

468 disk_parser.add_argument(

469 "-s",

470 "--simulate",

471 action="store_true",

472 help="Report disk info, but don't run tests",

473 )

474

475 cpu_parser.set_defaults(func=stress_cpu)

476 memory_parser.set_defaults(func=stress_memory)

477 disk_parser.set_defaults(func=stress_disk)

478

479 args = parser.parse_args()

480

481 if shutil.which("stress-ng") is None:

482 print("** The stress-ng utility is not installed; exiting!")

483 return 1

484 if not os.geteuid() == 0:

485 print("** This program must be run as root (or via sudo); exiting!")

486 return 1

487

488 retval = args.func(args)

489 print("retval is {}".format(retval))

490 print("*" * 62)

491 if retval == 0:

492 print("* stress-ng test passed!")

493 else:

494 print("** stress-ng test failed!")

495 print("*" * 62)

496

497 return retval

498

499

500 if __name__ == "__main__":

501 sys.exit(main())