1#!/usr/bin/env python3

2"""

3 Copyright (C) 2012-2015 Canonical Ltd.

4

5 Authors

6 Jeff Marcom <jeff.marcom@canonical.com>

7 Daniel Manrique <roadmr@ubuntu.com>

8 Jeff Lane <jeff@ubuntu.com>

9

10 This program is free software: you can redistribute it and/or modify

11 it under the terms of the GNU General Public License version 3,

12 as published by the Free Software Foundation.

13

14 This program is distributed in the hope that it will be useful,

15 but WITHOUT ANY WARRANTY; without even the implied warranty of

16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

17 GNU General Public License for more details.

18

19 You should have received a copy of the GNU General Public License

20 along with this program. If not, see <http://www.gnu.org/licenses/>.

21"""

22

23 from argparse import ArgumentParser, RawTextHelpFormatter

24 import datetime

25 import fcntl

26 import ipaddress

27 import logging

28 import math

29 import os

30 import re

31 import shlex

32 import socket

33 import struct

34 import subprocess

35 import tempfile

36 import threading

37 from subprocess import (

38 CalledProcessError,

39 check_call,

40 check_output,

41 STDOUT,

42 DEVNULL,

43)

44 from subprocess import run as sp_run

45 from contextlib import contextmanager, suppress

46 from pathlib import Path

47 import sys

48 import time

49

50# Global results[] variable to pass results from multiple threads....

51 results = []

52

53

54 class IPerfPerformanceTest(object):

55 """Measures performance of interface using iperf client

56 and target. Calculated speed is measured against theorectical

57 throughput of selected interface"""

58

59 def __init__(

60 self,

61 interface,

62 target,

63 fail_threshold,

64 cpu_load_fail_threshold,

65 iperf3,

66 num_threads,

67 reverse,

68 protocol="tcp",

69 data_size="1",

70 run_time=None,

71 scan_timeout=3600,

72 iface_timeout=120,

73 ):

74

75 self.iface = Interface(interface)

76 self.interface = interface

77 self.target = target

78 self.protocol = protocol

79 self.fail_threshold = fail_threshold

80 self.cpu_load_fail_threshold = cpu_load_fail_threshold

81 self.iperf3 = iperf3

82 self.num_threads = num_threads

83 self.data_size = data_size

84 self.run_time = run_time

85 self.scan_timeout = scan_timeout

86 self.iface_timeout = iface_timeout

87 self.reverse = reverse

88

89 def run_one_thread(self, cmd, port_num):

90 """Run a single test thread, storing the output in the global results[]

91 variable."""

92 cmd = cmd + " -p {}".format(port_num)

93 logging.debug("Executing command {}".format(cmd))

94 logging.info("Connecting to port {} on server....".format(port_num))

95 try:

96 iperf_return = check_output(

97 shlex.split(cmd),

98 stderr=subprocess.STDOUT,

99 universal_newlines=True,

100 )

101 except CalledProcessError as iperf_exception:

102 if iperf_exception.returncode != 124:

103 # timeout command will return 124 if iperf timed out, so any

104 # other return value means something did fail

105 if "unable to connect to server" in iperf_exception.output:

106 logging.error(

107 "Unable to connect to server on port {}".format(

108 port_num

109 )

110 )

111 if port_num == 5202:

112 # 5202 is 2nd port in high-speed configs

113 logging.warning("Your iperf3 server is not configured")

114 logging.warning("for high-speed network testing. See")

115 logging.warning("the Self-Test Guide's 'Network")

116 logging.warning("Performance Tuning' appendix for")

117 logging.warning("more information.")

118 else:

119 # Unknown error; log it....

120 logging.error(

121 "Failed executing iperf on port {}.".format(port_num)

122 )

123 logging.error(

124 "Output is '{}'".format(iperf_exception.output)

125 )

126 return iperf_exception.returncode

127 else:

128 # this is normal so we "except" this exception and we

129 # "pass through" whatever output iperf did manage to produce.

130 # When confronted with SIGTERM iperf should stop and output

131 # a partial (but usable) result.

132 logging.warning("iperf timed out - this should be OK")

133 iperf_return = iperf_exception.output

134 results.append(iperf_return)

135

136 def summarize_speeds(self):

137 """Search the global results[] variable, computing the throughput for

138 each thread and returning the total throughput for all threads."""

139 total_throughput = 0

140 n = 0

141 for run in results:

142 logging.debug(run)

143 # iperf3 provides "sender" and "receiver" summaries; remove them

144 run = re.sub(r".*(sender|receiver)", "", run)

145 speeds = list(map(float, re.findall(r"([\w\.]+)\sMbits/sec", run)))

146 if len(speeds) > 0:

147 total_throughput = total_throughput + sum(speeds) / len(speeds)

148 logging.debug(

149 "Throughput for thread {} is {}".format(

150 n, sum(speeds) / len(speeds)

151 )

152 )

153 logging.debug(

154 "Min Transfer speed for thread {}: {} Mb/s".format(

155 n, min(speeds)

156 )

157 )

158 logging.debug(

159 "Max Transfer speed for thread {}: {} Mb/s".format(

160 n, max(speeds)

161 )

162 )

163 n = n + 1

164 return total_throughput

165

166 def summarize_cpu(self):

167 """Return the average CPU load of all the threads, as reported by

168 iperf3. (Version 2 of iperf does not return CPU loads, in which case

169 this function returns 0.)"""

170 sum_cpu = 0.0

171 avg_cpu = 0.0

172 n = 0

173 for thread_results in results:

174 # "CPU Utilization" line present only in iperf3 output

175 new_cpu = re.findall(

176 r"CPU Utilization.*local/sender\s([\w\.]+)", thread_results

177 )

178 if new_cpu:

179 float_cpu = float(new_cpu[0])

180 logging.debug(

181 "CPU load for thread {}: {}%".format(n, float_cpu)

182 )

183 sum_cpu = sum_cpu + float_cpu

184 n = n + 1

185 if n > 0:

186 avg_cpu = sum_cpu / n

187 return avg_cpu

188

189 def find_numa(self, device):

190 """Return the NUMA node of the specified network device."""

191 filename = "/sys/class/net/" + device + "/device/numa_node"

192 try:

193 with open(filename, "r") as file:

194 node_num = int(file.read())

195 except FileNotFoundError:

196 node_num = -1

197 # Some systems (that don't support NUMA?) produce a node_num of -1.

198 # Later in the script, this will be interpreted to omit the -A option

199 # to iperf3, thus disabling NUMA features.

200 if node_num == -1:

201 logging.warning(

202 "WARNING: Could not find the NUMA node "

203 "associated with {}!".format(device)

204 )

205 else:

206 logging.info("NUMA node of {} is {}....".format(device, node_num))

207 return node_num

208

209 def extract_core_list(self, line):

210 """Extract a list of CPU cores from a line of the form:

211 NUMA node# CPU(s): a-b[,c-d[,...]]"""

212 colon = line.find(":")

213 cpu_list = line[colon + 1 :]

214 core_list = []

215 for core_range in cpu_list.split(","):

216 # core_range should be of the form "a-b" or "a"

217 range_list = core_range.split("-")

218 if len(range_list) > 1:

219 # core_range was a range ("a-b"), so we must add all cores in

220 # that range....

221 for i in range(int(range_list[0]), int(range_list[1]) + 1):

222 core_list.append(i)

223 elif len(range_list) == 1:

224 # core_range was a single CPU, so add just it....

225 core_list.append(int(range_list[0]))

226 else:

227 # Weirdness, so alert the user....

228 logging.error("Cannot parse CPU list:")

229 logging.error(cpu_list)

230 logging.debug("Will use CPU cores: {}....".format(core_list))

231 return core_list

232

233 def find_cores(self, numa_node):

234 """Return a list of CPU cores tied to the specified NUMA node."""

235 numa_return = check_output(

236 "lscpu", universal_newlines=True, stderr=STDOUT

237 ).split("\n")

238 # Note: If numa_node = -1, the below will never find a match, so

239 # core_list will remain empty, and later in the script, the -A option

240 # to iperf3 will be dropped.

241 expression = "NUMA node.*" + str(numa_node) + ".*CPU"

242

243 regex = re.compile(expression)

244 core_list = []

245 if numa_return:

246 for i in numa_return:

247 hit = regex.search(i)

248 if hit:

249 core_list = self.extract_core_list(i)

250 return core_list

251

252 def run(self):

253 # if max_speed is 0, assume it's wifi and move on

254 if self.iface.max_speed == 0:

255 logging.warning(

256 "No max speed detected, assuming Wireless device "

257 "and continuing with test."

258 )

259

260 threads = self.num_threads

261 if threads == 1:

262 logging.info("Using 1 thread.")

263 else:

264 logging.info("Using {} threads.".format(threads))

265

266 # Alter variables for iperf (2) vs. iperf3 -- Use iperf (2)'s own

267 # built-in threading, vs. this script's threading for iperf3. (Note

268 # that even with iperf 2, this script creates one separate thread

269 # for running iperf -- but only one; within that thread, iperf 2's

270 # own multi-threading handles that detail.)

271 if self.iperf3:

272 self.executable = "iperf3 -V"

273 start_port = 5201

274 iperf_threads = 1

275 python_threads = threads

276 node = self.find_numa(self.interface)

277 core_list = self.find_cores(node)

278 else:

279 self.executable = "iperf"

280 start_port = 5001

281 iperf_threads = threads

282 python_threads = 1

283

284 # IN THEORY, limiting the per-thread bit rate should help spread the

285 # load across all the threads and prevent huge discrepancies in CPU

286 # load, which can lead to test failures. IN PRACTICE, doing so helps,

287 # but there are still big differences in per-thread CPU load.

288 # Boost the theoretical exact split a bit so that one thread can take

289 # up a little slack if another falls behind.

290 thread_bit_rate = int(self.iface.max_speed / threads) + 1000

291 logging.debug("thread_bit_rate is {}".format(thread_bit_rate))

292

293 # If we set run_time, use that instead to build the command.

294 if self.run_time is not None:

295 cmd = "{} -b {}M -c {} -t {} -i 1 -f m -P {}".format(

296 self.executable,

297 thread_bit_rate,

298 self.target,

299 self.run_time,

300 iperf_threads,

301 )

302 if self.reverse:

303 cmd += " -R"

304 else:

305 # Because we can vary the data size, we need to vary the timeout as

306 # well. It takes an estimated 15 minutes to send 1GB over 10Mb/s.

307 # 802.11b is 11 Mb/s. So we'll assume 1.2x15 minutes or 18 minutes

308 # or 1080 seconds per Gigabit. This will allow for a long period of

309 # time without timeout to catch devices that slow down, and also

310 # not prematurely end iperf on low-bandwidth devices.

311 self.timeout = 1080 * int(self.data_size)

312 cmd = "timeout -k 1 {} {} -b {}M -c {} -n {}G -i 1 -f m -P {}".format( # noqa: E501

313 self.timeout,

314 self.executable,

315 thread_bit_rate,

316 self.target,

317 self.data_size,

318 iperf_threads,

319 )

320

321 # Handle threading -- start Python threads (even if just one is

322 # used), then use join() to wait for them all to complete....

323 t = []

324 results.clear()

325 for thread_num in range(0, python_threads):

326 if self.iperf3 and len(core_list) > 0:

327 core = core_list[thread_num % len(core_list)]

328 full_cmd = cmd + " -A {}".format(core)

329 else:

330 full_cmd = cmd

331 port_num = start_port + thread_num

332 t.append(

333 threading.Thread(

334 target=self.run_one_thread, args=(full_cmd, port_num)

335 )

336 )

337 t[thread_num].start()

338 for thread_num in range(0, python_threads):

339 t[thread_num].join()

340

341 throughput = self.summarize_speeds()

342 invalid_speed = False

343 try:

344 percent = throughput / int(self.iface.max_speed) * 100

345 except (ZeroDivisionError, TypeError):

346 # Catches a condition where the interface functions fine but

347 # ethtool fails to properly report max speed. In this case

348 # it's up to the reviewer to pass or fail.

349 percent = 0

350 invalid_speed = True

351 logging.info("Avg Transfer speed: {} Mb/s".format(throughput))

352 if invalid_speed:

353 # If we have no link_speed (e.g. wireless interfaces don't

354 # report this), then we shouldn't penalize them because

355 # the transfer may have been reasonable. So in this case,

356 # we'll exit with a pass-warning.

357 logging.warning("Unable to obtain maximum speed.")

358 logging.warning("Considering the test as passed.")

359 return 0

360 # Below is guaranteed to not throw an exception because we'll

361 # have exited above if it did.

362 logging.info(

363 "{:03.2f}% of theoretical max {} Mb/s".format(

364 percent, int(self.iface.max_speed)

365 )

366 )

367

368 if self.iperf3:

369 cpu_load = self.summarize_cpu()

370 logging.info(

371 "Average CPU utilization: {}%".format(round(cpu_load, 1))

372 )

373 else:

374 cpu_load = 0

375 if (

376 percent < self.fail_threshold

377 or cpu_load > self.cpu_load_fail_threshold

378 ):

379 logging.warning(

380 "The network test against {} failed because:".format(

381 self.target

382 )

383 )

384 if percent < self.fail_threshold:

385 logging.error(" Transfer speed: {} Mb/s".format(throughput))

386 logging.error(

387 " {:03.2f}% of theoretical max {} Mb/s\n".format(

388 percent, int(self.iface.max_speed)

389 )

390 )

391 if cpu_load > self.cpu_load_fail_threshold:

392 logging.error(" CPU load: {}%".format(cpu_load))

393 logging.error(

394 " CPU load is above {}% maximum\n".format(

395 self.cpu_load_fail_threshold

396 )

397 )

398 return 30

399

400 logging.debug("Passed benchmark against {}".format(self.target))

401

402 def optimize_num_threads(self):

403 """Find the approximate optimal number of threads."""

404 logging.info(" Optimizing Number of Threads ".center(60, "-"))

405 orig_run_time = self.run_time

406 orig_scan_timeout = self.scan_timeout

407 orig_num_threads = self.num_threads

408 self.run_time = 60

409 self.scan_timeout = 30

410 multiples = [0.5, 1, 1.5, 2]

411 max_throughput = 0

412 max_multiple = 0.5

413 for multiple in multiples:

414 self.num_threads = int(orig_num_threads * multiple)

415 logging.info(

416 "Testing optimization with {} threads".format(self.num_threads)

417 )

418 # Disable logging for the test runs that determine the optimum

419 # number of threads, since the output becomes too cluttered and

420 # confusing if we don't do so....

421 logger = logging.getLogger()

422 logger.disabled = True

423 self.run()

424 logger.disabled = False

425 throughput = self.summarize_speeds()

426 logging.info(

427 "Found throughput of {} with {} threads".format(

428 int(throughput), self.num_threads

429 )

430 )

431 if throughput > max_throughput:

432 max_throughput = throughput

433 max_multiple = multiple

434 self.run_time = orig_run_time

435 self.scan_timeout = orig_scan_timeout

436 self.num_threads = int(max_multiple * orig_num_threads)

437 logging.info(

438 "Setting number of threads to {}.".format(self.num_threads)

439 )

440

441

442 class StressPerformanceTest:

443

444 def __init__(self, interface, target, iperf3):

445 self.interface = interface

446 self.target = target

447 self.iperf3 = iperf3

448

449 def run(self):

450 if self.iperf3:

451 iperf_cmd = "timeout -k 1 320 iperf3 -c {} -t 300".format(

452 self.target

453 )

454 else:

455 iperf_cmd = "timeout -k 1 320 iperf -c {} -t 300".format(

456 self.target

457 )

458 print("Running iperf...")

459 iperf = subprocess.Popen(shlex.split(iperf_cmd))

460

461 ping_cmd = "ping -I {} {}".format(self.interface, self.target)

462 ping = subprocess.Popen(shlex.split(ping_cmd), stdout=subprocess.PIPE)

463 iperf.communicate()

464

465 ping.terminate()

466 (out, err) = ping.communicate()

467

468 if iperf.returncode != 0:

469 return iperf.returncode

470

471 print("Running ping test...")

472 result = 0

473 time_re = re.compile(r"(?<=time=)[0-9]*")

474 for line in out.decode().split("\n"):

475 time = time_re.search(line)

476

477 if time and int(time.group()) > 2000:

478 print(line)

479 print("ICMP packet was delayed by > 2000 ms.")

480 result = 1

481 if "unreachable" in line.lower():

482 print(line)

483 result = 1

484

485 return result

486

487

488 class Interface(socket.socket):

489 """

490 Simple class that provides network interface information.

491 """

492

493 def __init__(self, interface):

494

495 super(Interface, self).__init__(socket.AF_INET, socket.IPPROTO_ICMP)

496

497 self.interface = interface

498

499 self.dev_path = os.path.join("/sys/class/net", self.interface)

500

501 def _read_data(self, type):

502 try:

503 return open(os.path.join(self.dev_path, type)).read().strip()

504 except OSError:

505 logging.warning("%s: Attribute not found", type)

506

507 @property

508 def ipaddress(self):

509 freq = struct.pack("256s", self.interface[:15].encode())

510

511 try:

512 nic_data = fcntl.ioctl(self.fileno(), 0x8915, freq)

513 except IOError:

514 logging.error("No IP address for %s", self.interface)

515 return None

516 return socket.inet_ntoa(nic_data[20:24])

517

518 @property

519 def netmask(self):

520 freq = struct.pack("256s", self.interface.encode())

521

522 try:

523 mask_data = fcntl.ioctl(self.fileno(), 0x891B, freq)

524 except IOError:

525 logging.error("No netmask for %s", self.interface)

526 return None

527 return socket.inet_ntoa(mask_data[20:24])

528

529 @property

530 def link_speed(self):

531 return int(self._read_data("speed"))

532

533 @property

534 def max_speed(self):

535 speeds = [0]

536 # parse ethtool output, look for things like:

537 # 100baseSX, 40000baseNX, 10000baseT

538 try:

539 ethinfo = check_output(

540 ["ethtool", self.interface],

541 universal_newlines=True,

542 stderr=STDOUT,

543 ).split(" ")

544 expression = r"(\\d+)(base)([A-Z]+)|(\d+)(Mb/s)"

545

546 regex = re.compile(expression)

547 if ethinfo:

548 for i in ethinfo:

549 hit = regex.search(i)

550 if hit:

551 speeds.append(int(re.sub(r"\D", "", hit.group(0))))

552 except CalledProcessError as e:

553 logging.error("ethtool returned an error!")

554 logging.error(e.output)

555 except FileNotFoundError:

556 logging.warning("ethtool not found! Trying mii-tool")

557 # Parse mii-tool data for max speed

558 # search for numbers in the line starting with 'capabilities'

559 # return largest number as max_speed

560 try:

561 info = check_output(

562 ["mii-tool", "-v", self.interface],

563 universal_newlines=True,

564 stderr=STDOUT,

565 ).split("\n")

566 regex = re.compile(r"(\d+)(base)([A-Z]+)")

567 speeds = [0]

568 for line in filter(lambda c: "capabilities" in c, info):

569 for s in line.split(" "):

570 hit = regex.search(s)

571 if hit:

572 speeds.append(int(re.sub(r"\D", "", hit.group(0))))

573 except FileNotFoundError:

574 logging.warning("mii-tool not found! Unable to get max speed")

575 except CalledProcessError as e:

576 logging.error("mii-tool returned an error!")

577 logging.error(e.output)

578 return max(speeds)

579

580 @property

581 def macaddress(self):

582 return self._read_data("address")

583

584 @property

585 def duplex_mode(self):

586 return self._read_data("duplex")

587

588 @property

589 def status(self):

590 return self._read_data("operstate")

591

592 @property

593 def device_name(self):

594 return self._read_data("device/label")

595

596 @property

597 def ifindex(self):

598 return self._read_data("ifindex")

599

600 @property

601 def iflink(self):

602 return self._read_data("iflink")

603

604 @property

605 def phys_switch_id(self):

606 return self._read_data("phys_switch_id")

607

608

609 def get_test_parameters(args, environ):

610 # Decide the actual values for test parameters, which can come

611 # from one of two possible sources: command-line

612 # arguments, or environment variables.

613 # - If command-line args were given, they take precedence

614 # - Next come environment variables, if set.

615

616 params = {"test_target_iperf": None}

617

618 # See if we have environment variables

619 for key in params.keys():

620 params[key] = os.environ.get(key.upper(), "")

621

622 # Finally, see if we have the command-line arguments that are the ultimate

623 # override.

624 if args.target:

625 params["test_target_iperf"] = args.target

626

627 return params

628

629

630 def can_ping(the_interface, test_target):

631 working_interface = False

632 num_loops = 0

633 while (not working_interface) and (num_loops < 48):

634 working_interface = True

635

636 try:

637 sp_run(

638 ["ping", "-I", the_interface, "-c", "1", test_target],

639 stdout=DEVNULL,

640 stderr=DEVNULL,

641 )

642 except CalledProcessError:

643 working_interface = False

644

645 if not working_interface:

646 time.sleep(5)

647 num_loops += 1

648

649 return working_interface

650

651

652 def run_test(args, test_target):

653 # Ensure that interface is fully up by waiting until it can

654 # ping the test server

655 logging.info("Testing {} against {}".format(args.interface, test_target))

656 if can_ping(args.interface, test_target):

657 logging.info(

658 "Have successfully pinged {} on {}".format(

659 test_target, args.interface

660 )

661 )

662 else:

663 logging.error(

664 "Can't ping test server {} on {}".format(

665 test_target, args.interface

666 )

667 )

668 return 1

669

670 # Execute requested networking test

671 if args.test_type.lower() == "iperf":

672 error_number = 0

673 iperf_benchmark = IPerfPerformanceTest(

674 args.interface,

675 test_target,

676 args.fail_threshold,

677 args.cpu_load_fail_threshold,

678 args.iperf3,

679 args.num_threads,

680 args.reverse,

681 )

682 if args.datasize:

683 iperf_benchmark.data_size = args.datasize

684 if args.runtime:

685 iperf_benchmark.run_time = args.runtime

686 run_num = 0

687 if iperf_benchmark.num_threads == -1:

688 # Below is a really crude initial guesstimate based on our

689 # initial testing. This number is optimized (to some extent)

690 # by calling optimize_num_threads() below....

691 iperf_benchmark.num_threads = math.ceil(

692 iperf_benchmark.iface.link_speed / 10000

693 )

694 if iperf_benchmark.num_threads > 2:

695 iperf_benchmark.optimize_num_threads()

696 while not error_number and run_num < args.num_runs:

697 run_num += 1

698 logging.info(" Test Run Number %s ".center(60, "-"), run_num)

699 error_number = iperf_benchmark.run()

700 logging.info("")

701 elif args.test_type.lower() == "stress":

702 stress_benchmark = StressPerformanceTest(

703 args.interface, test_target, args.iperf3

704 )

705 error_number = stress_benchmark.run()

706 else:

707 logging.error("Unknown test type {}".format(args.test_type))

708 return 10

709 return error_number

710

711

712 def make_target_list(iface, test_targets, log_warnings):

713 """Convert comma-separated string of test targets into a list form.

714

715 Converts test target list in string form into Python list form, omitting

716 entries that are not on the current network segment.

717 :param iface:

718 Name of network interface device (eth0, etc.)

719 :param test_targets:

720 Input test targets as string of comma-separated IP addresses or

721 hostnames

722 :param emit_warnings:

723 Whether to log warning messages

724 :returns:

725 List form of input string, minus invalid values

726 """

727 test_targets_list = test_targets.split(",")

728 try:

729 net = ipaddress.IPv4Network(

730 "{}/{}".format(

731 Interface(iface).ipaddress, Interface(iface).netmask

732 ),

733 False,

734 )

735 except ipaddress.AddressValueError as e:

736 logging.error("Device {}: Invalid IP Address".format(iface))

737 logging.error(" {}".format(e))

738 logging.error("Aborting test now")

739 sys.exit(1)

740 first_addr = net.network_address + 1

741 last_addr = first_addr + net.num_addresses - 2

742 return_list = list(test_targets_list)

743 for test_target in test_targets_list:

744 try:

745 test_target_ip = socket.gethostbyname(test_target)

746 except OSError:

747 test_target_ip = test_target

748 if test_target_ip != "0.0.0.0":

749 try:

750 target = ipaddress.IPv4Address(test_target_ip)

751 if (target < first_addr) or (target > last_addr):

752 if log_warnings:

753 logging.warning(

754 "Removing iperf server {} ({}) from ".format(

755 test_target, target

756 )

757 )

758 logging.warning(

759 "test list since it's not within {}.".format(net)

760 )

761 return_list.remove(test_target)

762 except ValueError:

763 if log_warnings:

764 logging.warning(

765 "Invalid address: {}; skipping".format(test_target)

766 )

767 return_list.remove(test_target)

768 return_list.reverse()

769 if return_list == [""]:

770 del return_list[0]

771 return return_list

772

773

774# Wait until the specified interface comes up, or until iface_timeout.

775 def wait_for_iface_up(iface, timeout):

776 deadline = time.time() + timeout

777

778 net_if = Interface(iface)

779 while time.time() < deadline:

780 if net_if.status == "up":

781 logging.debug("Interface {} is up!".format(iface))

782 return True

783 logging.debug("Interface {} not yet up; waiting....".format(iface))

784 # Sleep whether or not interface is up because sometimes the IP

785 # address gets assigned after "ip" claims it's up.

786 time.sleep(5)

787

788 return False

789

790

791 def get_network_ifaces():

792 network_info = {}

793

794 for iface in Path("/sys/class/net/").glob("*"):

795 if ( 795 ↛ 800 line 795 didn't jump to line 800 because the condition on line 795 was never true

796 iface.name == "lo"

797 or iface.name.startswith("virbr")

798 or iface.name.startswith("lxdbr")

799 ):

800 continue

801 logging.debug("Retrieve the network attribute for %s interface", iface)

802 network_if = Interface(iface)

803 network_info[iface.name] = {

804 "status": network_if.status,

805 "phys_switch_id": network_if.phys_switch_id,

806 "iflink": network_if.iflink,

807 "ifindex": network_if.ifindex,

808 }

809

810 return network_info

811

812

813 def turn_down_network(iface):

814 logging.debug("Shutting down interface:%s", iface)

815 try:

816 check_call(["ip", "link", "set", "dev", iface, "down"])

817 return True

818 except CalledProcessError as interface_failure:

819 logging.error("Failed to shut down %s:%s", iface, interface_failure)

820 return False

821

822

823 def turn_up_network(iface, timeout):

824 logging.debug("Restoring interface:%s", iface)

825 try:

826 check_call(["ip", "link", "set", "dev", iface, "up"])

827 if not wait_for_iface_up(iface, timeout):

828 return False

829 return True

830 except CalledProcessError as interface_failure:

831 logging.error("Failed to restore %s:%s", iface, interface_failure)

832 return False

833

834

835 def check_underspeed(iface):

836 # Check for an underspeed link and abort if found,

837 # UNLESS --underspeed-ok option was used or max_speed is 0

838 # (which indicates a probable WiFi link)

839 network_if = Interface(iface)

840 if (

841 network_if.link_speed < network_if.max_speed

842 and network_if.max_speed != 0

843 ):

844 logging.error(

845 "Detected link speed ({}) is lower than detected max "

846 "speed ({})".format(network_if.link_speed, network_if.max_speed)

847 )

848 logging.error("Check your device configuration and try again.")

849 logging.error(

850 "If you want to override and test despite this "

851 "under-speed link, use"

852 )

853 logging.error("the --underspeed-ok option.")

854 return True

855 return False

856

857

858 def setup_network_ifaces(

859 network_info, target_network, underspeed_ok, toggle_status, timeout

860):

861 logging.debug("Setup network interface")

862

863 target_if_attrs = network_info.pop(target_network)

864 # bring up target interface

865 if target_if_attrs["status"] == "down" and not turn_up_network(

866 target_network, timeout

867 ):

868 raise SystemExit(

869 "Failed to bring up {} interface".format(target_network)

870 )

871

872 if not underspeed_ok and check_underspeed(target_network):

873 raise SystemExit(

874 "the network speed is incorrect for {} interface".format(

875 target_network

876 )

877 )

878

879 if (

880 target_if_attrs["phys_switch_id"] is not None

881 and target_if_attrs["ifindex"] != target_if_attrs["iflink"]

882 ):

883 # Means it's not a Physical network, but a Linux DSA network

884 # Reference:

885 # https://www.kernel.org/doc/Documentation/ABI/testing/sysfs-class-net

886 conduit_net = [

887 iface

888 for iface, attr in network_info.items()

889 if attr["ifindex"] == target_if_attrs["iflink"]

890 ]

891 if not conduit_net:

892 raise SystemExit("Conduit network interface not found")

893

894 conduit_net = conduit_net[0]

895 conduit_if_attrs = network_info.pop(conduit_net)

896

897 # bring up conduit network interface as well

898 if conduit_if_attrs["status"] == "down" and not turn_up_network(

899 conduit_net, timeout

900 ):

901 raise SystemExit(

902 "Failed to bring up {} conduit interface".format(conduit_net)

903 )

904

905 if toggle_status: 905 ↛ exit line 905 didn't return from function 'setup_network_ifaces' because the condition on line 905 was always true

906 # Shutdown other network interfaces

907 for iface, attrs in network_info.items():

908 if attrs["status"] == "up" and not turn_down_network(iface):

909 raise SystemExit(

910 "Failed to shutdown {} interface".format(iface)

911 )

912

913

914 def restore_network_ifaces(cur_network_info, origin_network_info, timeout):

915 status = True

916

917 logging.debug("Restoring interface")

918 for iface, attrs in origin_network_info.items():

919 if attrs["status"] != cur_network_info[iface]["status"]: 919 ↛ 918 line 919 didn't jump to line 918 because the condition on line 919 was always true

920 if attrs["status"] == "up" and not turn_up_network(iface, timeout): 920 ↛ 921 line 920 didn't jump to line 921 because the condition on line 920 was never true

921 status = False

922 elif attrs["status"] == "down" and not turn_down_network(iface):

923 status = False

924

925 return status

926

927

928@contextmanager

929 def interface_test_initialize(

930 target_dev, underspeed_ok, dont_toggle_ifaces, recover_timeout

931):

932 tempfile_route = tempfile.TemporaryFile()

933 try:

934 network_info = get_network_ifaces()

935 # Back up routing table, since network down/up process

936 # tends to trash it....

937 logging.debug("Backup routing table")

938 check_call(

939 ["ip", "route", "save", "table", "all"], stdout=tempfile_route

940 )

941 setup_network_ifaces(

942 network_info,

943 target_dev,

944 underspeed_ok,

945 not dont_toggle_ifaces,

946 recover_timeout,

947 )

948

949 yield

950

951 finally:

952 cur_network_info = get_network_ifaces()

953 recover_success = restore_network_ifaces(

954 cur_network_info, network_info, recover_timeout

955 )

956

957 # Restore routing table to original state

958 logging.debug("Restore routing table")

959 with suppress(CalledProcessError):

960 # Harmless "RTNETLINK answers: File exists" messages on stderr

961

962 # This always errors out -- but it works!

963 # The problem is virbr0, which has the "linkdown" flag, which the

964 # "ip route restore" command can't handle.

965 sp_run(

966 ["ip", "route", "restore"],

967 stdin=tempfile_route,

968 stderr=DEVNULL,

969 )

970

971 if not recover_success:

972 raise CalledProcessError(3, "restore network failed")

973

974

975 def interface_test(args):

976 if not ("test_type" in vars(args)):

977 return

978

979 # Get the actual test data from one of two possible sources

980 test_parameters = get_test_parameters(args, os.environ)

981

982 if args.test_type.lower() == "iperf" or args.test_type.lower() == "stress": 982 ↛ 989 line 982 didn't jump to line 989 because the condition on line 982 was always true

983 test_targets = test_parameters["test_target_iperf"]

984 test_targets_list = make_target_list(

985 args.interface, test_targets, True

986 )

987

988 # Validate that we got reasonable values

989 if not test_targets_list or "example.com" in test_targets:

990 # Default values found in config file

991 logging.error("Valid target server has not been supplied.")

992 logging.error(

993 "Configuration settings can be configured 3 different " "ways:"

994 )

995 logging.error(

996 "1- If calling the script directly, pass the --target " "option"

997 )

998 logging.error("2- Define the TEST_TARGET_IPERF environment variable")

999 logging.error(

1000 "3- If running the test via checkbox/plainbox, define " "the "

1001 )

1002 logging.error("target in /etc/xdg/canonical-certification.conf")

1003 logging.error(

1004 "Please run this script with -h to see more details on "

1005 "how to configure"

1006 )

1007 sys.exit(1)

1008

1009 # Testing begins here!

1010 try:

1011 error_number = 1

1012 with interface_test_initialize(

1013 args.interface,

1014 args.underspeed_ok,

1015 args.dont_toggle_ifaces,

1016 args.iface_timeout,

1017 ):

1018 logging.debug(

1019 "Start Iperf testing with %s iperf server", test_targets_list

1020 )

1021 start_time = datetime.datetime.now()

1022 first_loop = True

1023 # Keep testing until a success

1024 # or we run out of both targets and time

1025 while test_targets_list: 1025 ↛ 1049 line 1025 didn't jump to line 1049

1026 test_target = test_targets_list.pop().strip()

1027 error_number = run_test(args, test_target)

1028 elapsed_seconds = (

1029 datetime.datetime.now() - start_time

1030 ).seconds

1031 if (

1032 elapsed_seconds > args.scan_timeout and not first_loop

1033 ) or not error_number:

1034 break

1035 if not test_targets_list: 1035 ↛ 1025 line 1035 didn't jump to line 1025 because the condition on line 1035 was always true

1036 logging.warning(

1037 " Exhausted test target list; trying again ".center(

1038 60, "="

1039 )

1040 )

1041 test_targets_list = make_target_list(

1042 args.interface, test_targets, False

1043 )

1044 time.sleep(

1045 30

1046 ) # Wait to give server(s) time to come online

1047 first_loop = False

1048

1049 return error_number

1050 except CalledProcessError:

1051 return 3

1052

1053

1054 def interface_info(args):

1055

1056 info_set = ""

1057 if "all" in vars(args):

1058 info_set = args.all

1059

1060 for key, value in vars(args).items():

1061 if value is True or info_set is True:

1062 key = key.replace("-", "_")

1063 try:

1064 print(

1065 key + ":",

1066 getattr(Interface(args.interface), key),

1067 file=sys.stderr,

1068 )

1069 except AttributeError:

1070 pass

1071

1072

1073 def main():

1074

1075 intro_message = """

1076 Network module

1077

1078 This script provides benchmarking and information for a specified network

1079 interface.

1080

1081 Example NIC information usage:

1082 network.py info -i eth0 --max-speed

1083

1084 For running iperf test:

1085 network.py test -i eth0 -t iperf --target 192.168.0.1

1086 NOTE: The iperf test requires an iperf server running on the same network

1087 segment that the test machine is running on.

1088

1089 Also, you can use iperf3 rather than iperf2 by specifying the -3 or --iperf3

1090 option like so:

1091

1092 network.py test -i eth0 -t iperf -3 --target 192.168.0.1

1093

1094 Configuration

1095=============

1096

1097 Configuration can be supplied in three different ways, with the following

1098 priorities:

1099

1100 1- Command-line parameters (see above).

1101 2- Environment variables (example will follow).

1102 3- If run via checkbox/plainbox, /etc/xdg/checkbox-certification.conf

1103 can have the below-mentioned environment variables defined in the

1104 [environment] section. An example file is provided and can be simply

1105 modified with the correct values.

1106

1107 Environment variables

1108=====================

1109 The variables are:

1110 TEST_TARGET_IPERF

1111

1112 example config file

1113===================

1114[environment]

1115 TEST_TARGET_IPERF = iperf-server.example.com

1116

1117

1118**NOTE**

1119

1120"""

1121

1122 parser = ArgumentParser(

1123 description=intro_message, formatter_class=RawTextHelpFormatter

1124 )

1125 subparsers = parser.add_subparsers()

1126

1127 # Main cli options

1128 test_parser = subparsers.add_parser(

1129 "test", help=("Run network performance test")

1130 )

1131 info_parser = subparsers.add_parser("info", help=("Gather network info"))

1132

1133 # Sub test options

1134 action = test_parser.add_mutually_exclusive_group()

1135

1136 test_parser.add_argument("-i", "--interface", type=str, required=True)

1137 test_parser.add_argument(

1138 "-t",

1139 "--test_type",

1140 type=str,

1141 choices=("iperf", "stress"),

1142 default="iperf",

1143 help=("[iperf *Default*]"),

1144 )

1145 test_parser.add_argument(

1146 "-3",

1147 "--iperf3",

1148 default=False,

1149 action="store_true",

1150 help=(

1151 "Tells the script to use iperf3 for testing, rather than the "

1152 "default of iperf2"

1153 ),

1154 )

1155 test_parser.add_argument("--target", type=str)

1156 action.add_argument(

1157 "--datasize",

1158 type=str,

1159 default="1",

1160 help=(

1161 "CANNOT BE USED WITH --runtime. Amount of data to send. For "

1162 "iperf tests this will direct iperf to send DATASIZE GB of "

1163 "data to the target."

1164 ),

1165 )

1166 action.add_argument(

1167 "--runtime",

1168 type=int,

1169 default=60,

1170 help=(

1171 "CANNOT BE USED WITH --datasize. Send data for *runtime* "

1172 "seconds. For iperf tests, this will send data for the amount "

1173 "of time indicated, rather than until a certain file size is "

1174 "reached."

1175 ),

1176 )

1177 test_parser.add_argument(

1178 "--scan-timeout",

1179 type=int,

1180 default=60,

1181 help=(

1182 "Sets the maximum time, in seconds, the test will scan for "

1183 "iperf servers before giving up."

1184 ),

1185 )

1186 test_parser.add_argument(

1187 "--iface-timeout",

1188 type=int,

1189 default=120,

1190 help=(

1191 "Sets the maximum time, in seconds, the test will wait for "

1192 "an interface to come up after a test before giving up."

1193 ),

1194 )

1195 test_parser.add_argument(

1196 "--config",

1197 type=str,

1198 default="/etc/checkbox.d/network.cfg",

1199 help="Supply config file for target/host network parameters",

1200 )

1201 test_parser.add_argument(

1202 "--fail-threshold",

1203 type=int,

1204 default=40,

1205 help=(

1206 "IPERF Test ONLY. Set the failure threshold (Percent of maximum "

1207 "theoretical bandwidth) as a number like 80. (Default is "

1208 "%(default)s)"

1209 ),

1210 )

1211 test_parser.add_argument(

1212 "--cpu-load-fail-threshold",

1213 type=int,

1214 default=100,

1215 help=(

1216 "(IPERF Test ONLY and meaningful ONLY with --iperf3. Set the "

1217 "failure threshold (above which the CPU load must not rise) as "

1218 "a number like 80. (Default is %(default)s)"

1219 ),

1220 )

1221 test_parser.add_argument(

1222 "--num_runs",

1223 type=int,

1224 default=1,

1225 help=("Number of times to run the test. (Default is %(default)s)"),

1226 )

1227 test_parser.add_argument(

1228 "--debug",

1229 default=False,

1230 action="store_true",

1231 help="Turn on verbose output",

1232 )

1233 test_parser.add_argument(

1234 "--underspeed-ok",

1235 default=False,

1236 action="store_true",

1237 help="Run test even if an underspeed 1ink is detected",

1238 )

1239 test_parser.add_argument(

1240 "--num-threads",

1241 type=int,

1242 default=-1,

1243 help=(

1244 "Number of threads to use in the test. "

1245 "(Default is computed based on network speed.)"

1246 ),

1247 )

1248 test_parser.add_argument(

1249 "--reverse",

1250 default=False,

1251 action="store_true",

1252 help="Run in reverse mode (server sends, client receives)",

1253 )

1254 test_parser.add_argument(

1255 "--dont-toggle-ifaces",

1256 default=False,

1257 action="store_true",

1258 help="Do not turn of other interfaces while testing.",

1259 )

1260

1261 # Sub info options

1262 info_parser.add_argument("-i", "--interface", type=str, required=True)

1263 info_parser.add_argument("--all", default=False, action="store_true")

1264 info_parser.add_argument(

1265 "--duplex-mode", default=False, action="store_true"

1266 )

1267 info_parser.add_argument(

1268 "--link-speed", default=False, action="store_true"

1269 )

1270 info_parser.add_argument("--max-speed", default=False, action="store_true")

1271 info_parser.add_argument("--ipaddress", default=False, action="store_true")

1272 info_parser.add_argument("--netmask", default=False, action="store_true")

1273 info_parser.add_argument(

1274 "--device-name", default=False, action="store_true"

1275 )

1276 info_parser.add_argument(

1277 "--macaddress", default=False, action="store_true"

1278 )

1279 info_parser.add_argument(

1280 "--status",

1281 default=False,

1282 action="store_true",

1283 help=("displays connection status"),

1284 )

1285 info_parser.add_argument(

1286 "--debug",

1287 default=False,

1288 action="store_true",

1289 help="Turn on verbose output",

1290 )

1291

1292 test_parser.set_defaults(func=interface_test)

1293 info_parser.set_defaults(func=interface_info)

1294

1295 args = parser.parse_args()

1296 if (

1297 args.func.__name__ is interface_test

1298 and not args.cpu_load_fail_threshold != 100

1299 and not args.iperf3

1300 ):

1301 parser.error(

1302 "--cpu-load-fail-threshold can only be set with " "--iperf3."

1303 )

1304

1305 if args.debug:

1306 logging.basicConfig(level=logging.DEBUG)

1307 else:

1308 logging.basicConfig(level=logging.INFO)

1309

1310 if "func" not in args:

1311 parser.print_help()

1312 else:

1313 return args.func(args)

1314

1315

1316 if __name__ == "__main__":

1317 sys.exit(main())