1#!/usr/bin/env python3

2

3"""

4 Script to test virtualization functionality

5

6 Copyright (C) 2013, 2014 Canonical Ltd.

7

8 Authors

9 Jeff Marcom <jeff.marcom@canonical.com>

10 Daniel Manrique <roadmr@ubuntu.com>

11 Jeff Lane <jeff@ubuntu.com>

12

13 This program is free software: you can redistribute it and/or modify

14 it under the terms of the GNU General Public License version 3,

15 as published by the Free Software Foundation.

16

17 This program is distributed in the hope that it will be useful,

18 but WITHOUT ANY WARRANTY; without even the implied warranty of

19 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

20 GNU General Public License for more details.

21

22 You should have received a copy of the GNU General Public License

23 along with this program. If not, see <http://www.gnu.org/licenses/>.

24

25"""

26 from argparse import ArgumentParser

27 import os

28 import logging

29 import requests

30 import shlex

31 from subprocess import (

32 Popen,

33 PIPE,

34 DEVNULL,

35 CalledProcessError,

36 check_output,

37 call,

38)

39 import sys

40 import tempfile

41 import tarfile

42 import time

43 import urllib.request

44 from urllib.parse import urlparse

45 from uuid import uuid4

46

47 DEFAULT_TIMEOUT = 500

48

49

50# The "TAR" type is a tarball that contains both

51# a disk image and a kernel binary. This is useful

52# on architectures that don't (yet) have a bootloader

53# in the disk image that we can chain to, and instead

54# we need to have qemu load boot files externally

55 CLOUD_IMAGE_TYPE_TAR = 1

56 CLOUD_IMAGE_TYPE_DISK = 2

57

58 QEMU_DISK_TYPE_SD = 1

59 QEMU_DISK_TYPE_VIRTIO = 2

60 QEMU_DISK_TYPE_VIRTIO_BLK = 3

61

62 QEMU_ARCH_CONFIG = {

63 "arm64": {

64 "cloudimg_type": CLOUD_IMAGE_TYPE_TAR,

65 "cloudimg_arch": "arm64",

66 "qemu_bin": "qemu-system-aarch64",

67 "qemu_disk_type": QEMU_DISK_TYPE_VIRTIO_BLK,

68 "qemu_extra_args": [

69 "-cpu",

70 "host",

71 "-enable-kvm",

72 ],

73 },

74 "armhf": {

75 "cloudimg_type": CLOUD_IMAGE_TYPE_TAR,

76 "cloudimg_arch": "armhf",

77 "qemu_bin": "qemu-system-arm",

78 "qemu_disk_type": QEMU_DISK_TYPE_VIRTIO_BLK,

79 "qemu_extra_args": [

80 "-machine",

81 "virt",

82 "-cpu",

83 "host",

84 "-enable-kvm",

85 "-serial",

86 "stdio",

87 ],

88 },

89 "amd64": {

90 "cloudimg_type": CLOUD_IMAGE_TYPE_DISK,

91 "cloudimg_arch": "amd64",

92 "qemu_bin": "qemu-system-x86_64",

93 "qemu_disk_type": QEMU_DISK_TYPE_VIRTIO,

94 "qemu_extra_args": [

95 "-machine",

96 "accel=kvm:tcg",

97 ],

98 },

99 "i386": {

100 "cloudimg_type": CLOUD_IMAGE_TYPE_DISK,

101 "cloudimg_arch": "i386",

102 "qemu_bin": "qemu-system-x86_64",

103 "qemu_disk_type": QEMU_DISK_TYPE_VIRTIO,

104 "qemu_extra_args": [

105 "-machine",

106 "accel=kvm:tcg",

107 ],

108 },

109 "ppc64el": {

110 "cloudimg_type": CLOUD_IMAGE_TYPE_DISK,

111 "cloudimg_arch": "ppc64el",

112 "qemu_bin": "qemu-system-ppc64",

113 "qemu_disk_type": QEMU_DISK_TYPE_VIRTIO,

114 "qemu_extra_args": [

115 "-enable-kvm",

116 "-machine",

117 "pseries,usb=off",

118 "-cpu",

119 "host",

120 ],

121 },

122 "s390x": {

123 "cloudimg_type": CLOUD_IMAGE_TYPE_DISK,

124 "cloudimg_arch": "s390x",

125 "qemu_bin": "qemu-system-s390x",

126 "qemu_disk_type": QEMU_DISK_TYPE_VIRTIO,

127 "qemu_extra_args": [

128 "-enable-kvm",

129 "-machine",

130 "s390-ccw-virtio",

131 ],

132 },

133}

134

135

136 def get_release_to_test():

137 try:

138 import distro

139

140 if distro.id() == "ubuntu-core":

141 return "{}.04".format(distro.version())

142 return distro.version()

143 except (ImportError, CalledProcessError):

144 import lsb_release

145

146 return lsb_release.get_distro_information()["RELEASE"]

147

148

149 def get_codename_to_test():

150 try:

151 import distro

152

153 if distro.id() == "ubuntu-core":

154 codename = "focal"

155 if distro.version() == "18":

156 codename = "bionic"

157 elif distro.version() == "16":

158 codename = "xenial"

159 return codename

160 return distro.codename().split()[0].lower()

161 except (ImportError, CalledProcessError):

162 import lsb_release

163

164 lsb_release.get_distro_information()["CODENAME"]

165

166

167 class QemuRunner(object):

168 def __init__(self, arch):

169 self.arch = arch

170 self.config = QEMU_ARCH_CONFIG[arch]

171 self.drive_id = 0

172 # Parameters common to all architectures

173 self.params = [

174 self.config["qemu_bin"],

175 "-m",

176 "1024",

177 "-display",

178 "none",

179 "-nographic",

180 "-net",

181 "nic",

182 "-net",

183 "user,net=10.0.0.0/8,host=10.0.0.1,hostfwd=tcp::2222-:22",

184 ]

185 # If arch is arm64, add the machine type for gicv3, or default to old

186 # type

187 if self.arch == "arm64":

188 (

189 self.config["qemu_extra_args"].extend(

190 ["-machine", "virt,gic_version=host"]

191 )

192 )

193 # Add any architecture-specific parameters

194 if "qemu_extra_args" in self.config:

195 self.params = self.params + self.config["qemu_extra_args"]

196

197 self.append = []

198 if self.config["cloudimg_type"] == CLOUD_IMAGE_TYPE_TAR:

199 self.append = self.append + [

200 "console=ttyAMA0",

201 "earlyprintk=serial",

202 "ro",

203 "rootfstype=ext4",

204 "root=LABEL=cloudimg-rootfs",

205 "rootdelay=10",

206 ]

207

208 def add_boot_files(self, kernel=None, initrd=None, dtb=None):

209 if kernel:

210 self.params = self.params + ["-kernel", kernel]

211 if initrd:

212 self.params = self.params + ["-initrd", initrd]

213 if dtb:

214 self.params = self.params + ["-dtb", dtb]

215

216 def add_drive(self, cloudimg):

217 drive = ["-drive"]

218 if self.config["qemu_disk_type"] == QEMU_DISK_TYPE_SD:

219 drive = drive + ["file=%s,if=sd,cache=writeback" % (cloudimg)]

220 elif self.config["qemu_disk_type"] == QEMU_DISK_TYPE_VIRTIO:

221 drive = drive + ["file=%s,if=virtio" % (cloudimg)]

222 elif self.config["qemu_disk_type"] == QEMU_DISK_TYPE_VIRTIO_BLK:

223 drive = drive + [

224 "file=%s,if=none,id=disk.%d" % (cloudimg, self.drive_id)

225 ]

226 drive = drive + [

227 "-device",

228 "virtio-blk-device,drive=disk.%d" % (self.drive_id),

229 ]

230 self.params = self.params + drive

231 self.drive_id = self.drive_id + 1

232

233 def get_params(self):

234 params = self.params

235 if self.append:

236 params = params + ["-append", '"%s"' % (" ".join(self.append))]

237 return params

238

239

240 class KVMTest(object):

241

242 def __init__(self, image=None, timeout=500, debug_file=None):

243 self.image = image

244 self.timeout = timeout

245 self.debug_file = debug_file

246 self.arch = check_output(

247 ["dpkg", "--print-architecture"], universal_newlines=True

248 ).strip()

249 self.qemu_config = QEMU_ARCH_CONFIG[self.arch]

250 self.release = get_codename_to_test()

251

252 def url_to_path(self, image_path):

253 """

254 Test the provided image path to determine if it's a URL or or a simple

255 file path

256 """

257 url = urlparse(image_path)

258 if url.scheme == "" or url.scheme == "file":

259 # Gives us path wheter we specify a filesystem path or a file URL

260 logging.debug("Cloud image exists locally at %s" % url.path)

261 return url.path

262 elif url.scheme == "http" or url.scheme == "ftp":

263 # Gives us the stuff needed to build the URL to download the image

264 return self.download_image(image_path)

265

266 def get_image_name_and_url(self, image_url=None):

267 """

268 Build a URL for official Ubuntu images hosted either at

269 cloud-images.ubuntu.com or on a maas server hosting a mirror of

270 cloud-images.ubuntu.com

271 """

272

273 def _construct_filename(alt_pattern=None, initial_url=None):

274 if self.qemu_config["cloudimg_type"] == CLOUD_IMAGE_TYPE_TAR:

275 cloud_iso = "%s-server-cloudimg-%s.tar.gz" % (

276 self.release,

277 self.qemu_config["cloudimg_arch"],

278 )

279 elif alt_pattern == "modern":

280 # LP 1635345 - yakkety and beyond have a new naming scheme

281 cloud_iso = "%s-server-cloudimg-%s.img" % (

282 self.release,

283 self.qemu_config["cloudimg_arch"],

284 )

285 elif self.qemu_config["cloudimg_type"] == CLOUD_IMAGE_TYPE_DISK:

286 cloud_iso = "%s-server-cloudimg-%s-disk1.img" % (

287 self.release,

288 self.qemu_config["cloudimg_arch"],

289 )

290 elif initial_url:

291 # LP 1662580 - if we pass a full URL, assume the last piece is

292 # the filname and return that.

293 cloud_iso = initial_url.split("/")[-1]

294 else:

295 logging.error("Unknown cloud image type")

296 sys.exit(1)

297

298 return cloud_iso

299

300 def _construct_url(initial_url, cloud_iso):

301 return "/".join((initial_url, cloud_iso))

302

303 def _test_cloud_url(url):

304 # test our URL to make sure it's reachable

305 try:

306 ret = requests.head(url)

307 except OSError as e:

308 logging.error("Unable to connect to {}".format(url))

309 logging.error(" * Message: {}".format(e.with_traceback(None)))

310 return False

311

312 if ret.status_code != 200:

313 return False

314 else:

315 return True

316

317 if image_url is None:

318 # If we have not specified a URL to get our images from, default

319 # to ubuntu.com

320 cloud_iso = _construct_filename()

321 initial_url = "/".join(

322 ("http://cloud-images.ubuntu.com", self.release, "current")

323 )

324 full_url = _construct_url(initial_url, cloud_iso)

325 # Test our URL and rebuild with alternate name

326 if not _test_cloud_url(full_url):

327 logging.warn("Cloud Image URL not valid: %s" % full_url)

328 logging.warn(

329 " * This means we could not reach the remote "

330 "file. Retrying with a different filename "

331 "schema."

332 )

333 cloud_iso = _construct_filename("modern")

334 full_url = _construct_url(initial_url, cloud_iso)

335 # retest one more time then exit if it still fails

336 if not _test_cloud_url(full_url):

337 logging.error("Cloud URL is not valid: %s" % full_url)

338 logging.error(

339 " * It appears that there is a problem "

340 "finding the expected file. Check the URL "

341 "noted above."

342 )

343 sys.exit(1)

344 else:

345 url = urlparse(image_url)

346 if (

347 url.path.endswith("/")

348 or url.path == ""

349 or not (

350 url.path.endswith(".img") or url.path.endswith(".tar.gz")

351 )

352 ):

353 # If we have a relative URL (local copies of official images)

354 # http://192.168.0.1/ or http://192.168.0.1/images/

355 cloud_iso = _construct_filename()

356 full_url = _construct_url(image_url.rstrip("/"), cloud_iso)

357 if not _test_cloud_url(full_url):

358 logging.warn("Cloud Image URL not valid: %s" % full_url)

359 logging.warn(

360 " * This means we could not reach the remote "

361 "file. Retrying with a different filename "

362 "schema."

363 )

364 cloud_iso = _construct_filename("modern")

365 full_url = _construct_url(image_url.rstrip("/"), cloud_iso)

366 if not _test_cloud_url(full_url):

367 logging.error("Cloud URL is not valid: %s" % full_url)

368 logging.error(

369 " * It appears that there is a problem "

370 "finding the expected file. Check the "

371 "URL noted above."

372 )

373 sys.exit(1)

374 else:

375 # Assume anything else is an absolute URL to a remote server

376 if not _test_cloud_url(image_url):

377 logging.error("Cloud Image URL invalid: %s" % image_url)

378 logging.error(" * Check the URL and ensure it is correct")

379 sys.exit(1)

380 else:

381 full_url = image_url

382 cloud_iso = _construct_filename(initial_url=full_url)

383

384 return full_url, cloud_iso

385

386 def download_image(self, image_url=None):

387 """

388 Downloads Cloud image for same release as host machine

389 """

390 if image_url is None:

391 full_url, cloud_iso = self.get_image_name_and_url()

392 else:

393 full_url, cloud_iso = self.get_image_name_and_url(image_url)

394 logging.debug("Acquiring cloud image from: {}".format(full_url))

395

396 # Attempt download

397 try:

398 urllib.request.urlretrieve(full_url, cloud_iso)

399 except (

400 IOError,

401 OSError,

402 urllib.error.HTTPError,

403 urllib.error.URLError,

404 ) as exception:

405 logging.error(

406 "Failed download of image from %s: %s", image_url, exception

407 )

408 return False

409

410 # Unpack img file from tar

411 if self.qemu_config["cloudimg_type"] == CLOUD_IMAGE_TYPE_TAR:

412 cloud_iso_tgz = tarfile.open(cloud_iso)

413 cloud_iso = cloud_iso.replace("tar.gz", "img")

414 cloud_iso_tgz.extract(cloud_iso)

415

416 if not os.path.isfile(cloud_iso):

417 return False

418

419 return cloud_iso

420

421 def boot_image(self, data_disk):

422 """

423 Attempts to boot the newly created qcow image using

424 the config data defined in config.iso

425 """

426

427 logging.debug("Attempting boot for:{}".format(data_disk))

428

429 qemu = QemuRunner(self.arch)

430

431 # Assume that a tar type image is not self-bootable, so

432 # therefore requires explicit bootfiles (otherwise, why

433 # not just use the disk format directly?

434 if self.qemu_config["cloudimg_type"] == CLOUD_IMAGE_TYPE_TAR:

435 for dir in ["/boot", "/"]:

436 kernel = os.path.join(dir, "vmlinuz")

437 initrd = os.path.join(dir, "initrd.img")

438 if os.path.isfile(kernel):

439 qemu.add_boot_files(kernel=kernel, initrd=initrd)

440 break

441

442 qemu.add_drive(data_disk)

443

444 # Should we attach the cloud config disk

445 if os.path.isfile("seed.iso"):

446 logging.debug("Attaching Cloud config disk")

447 qemu.add_drive("seed.iso")

448

449 params = qemu.get_params()

450 logging.debug("Using params:{}".format(" ".join(params)))

451

452 logging.info(

453 "Storing VM console output in {}".format(

454 os.path.realpath(self.debug_file)

455 )

456 )

457 # Open VM STDERR/STDOUT log file for writing

458 try:

459 file = open(self.debug_file, "w")

460 except IOError:

461 logging.error("Failed creating file:{}".format(self.debug_file))

462 return False

463

464 # Start Virtual machine

465 self.process = Popen(

466 params,

467 stdin=PIPE,

468 stderr=file,

469 stdout=file,

470 universal_newlines=True,

471 shell=False,

472 )

473

474 def create_cloud_disk(self):

475 """

476 Generate Cloud meta data and creates an iso object

477 to be mounted as virtual device to instance during boot.

478 """

479 for file in ["user-data", "meta-data"]:

480 logging.debug("Creating cloud %s", file)

481 with open(file, "wt") as data_file:

482 os.fchmod(data_file.fileno(), 0o777)

483 data_file.write(vars()[file.replace("-", "_")])

484

485 # Create Data ISO hosting user & meta cloud config data

486 try:

487 check_output(

488 [

489 "genisoimage",

490 "-output",

491 "seed.iso",

492 "-volid",

493 "cidata",

494 "-joliet",

495 "-rock",

496 "user-data",

497 "meta-data",

498 ],

499 universal_newlines=True,

500 )

501 except CalledProcessError:

502 logging.exception("Cloud data disk creation failed")

503

504 def log_check(self, stream):

505 if "CERTIFICATION BOOT COMPLETE" in stream:

506 return 0

507 else:

508 return 1

509

510 def start(self):

511 if self.arch == "arm64":

512 # lp:1548539 - For arm64, we need to make sure we're using qemu

513 # later than 2.0.0 to enable gic_version functionality

514 logging.debug("Checking QEMU version for arm64 arch")

515 cmd = "apt-cache policy qemu-system-arm | grep Installed"

516 installed_version = (

517 check_output(["/bin/bash", "-c", cmd])

518 .decode()

519 .split(":", 1)[1]

520 .strip()

521 )

522

523 cmd = 'dpkg --compare-versions "2.0.0" "lt" "{}"'.format(

524 installed_version

525 )

526 retcode = call(["/bin/bash", "-c", cmd])

527 if retcode != 0:

528 logging.error(

529 "arm64 needs qemu-system version later than " "2.0.0"

530 )

531 return 1

532

533 logging.debug("Starting KVM Test")

534 status = 1

535 # Create temp directory:

536

537 date = time.strftime("%b_%d_%Y_")

538 with tempfile.TemporaryDirectory("_kvm_test", date) as temp_dir:

539

540 os.chmod(temp_dir, 0o744)

541 os.chdir(temp_dir)

542 if not self.image:

543 logging.debug("No image specified, downloading one now.")

544 # Download cloud image

545 self.image = self.download_image()

546 else:

547 logging.debug(

548 "Cloud image location specified: %s" % self.image

549 )

550 self.image = self.url_to_path(self.image)

551

552 if self.image and os.path.isfile(self.image):

553

554 if "cloud" in self.image:

555 # Will assume we need to supply cloud meta data

556 # for instance boot to be successful

557 self.create_cloud_disk()

558

559 # Boot Virtual Machine

560 self.boot_image(self.image)

561

562 # If running in console, reset console window to regain

563 # control from VM Serial I/0

564 if sys.stdout.isatty():

565 call("reset")

566 # Check to be sure VM boot was successful

567 self.elapsed_time = 0

568 status = 1

569 while self.elapsed_time <= self.timeout:

570 # Check log every 30 seconds to see if the VM boots

571 with open(self.debug_file, "r") as debug_file:

572 status = self.log_check(debug_file.read())

573 if status == 0:

574 logging.info("Booted successfully.")

575 break

576 else:

577 # Sleep 30 seconds and log check again

578 time.sleep(30)

579 self.elapsed_time += 30

580 else:

581 # Finally, if we didn't get the Success message by now try

582 # one more time and return 1 if we still haven't booted

583 if status != 0:

584 with open(self.debug_file, "r") as debug_file:

585 stream = debug_file.read()

586 status = self.log_check(stream)

587 if status == 0:

588 logging.info("Booted successfully.")

589 else:

590 logging.error("KVM instance failed to boot.")

591 logging.error("Console output".center(72, "="))

592 logging.error(stream)

593 self.process.terminate()

594 elif not self.image:

595 logging.error("Could not find downloaded image")

596 else:

597 logging.error("Could not find: {}".format(self.image))

598

599 return status

600

601

602 class RunCommand(object):

603 """

604 Runs a command and can return all needed info:

605 * stdout

606 * stderr

607 * return code

608 * original command

609

610 Convenince class to avoid the same repetitive code to run shell commands.

611 """

612

613 def __init__(self, cmd=None):

614 self.stdout = None

615 self.stderr = None

616 self.returncode = None

617 self.cmd = cmd

618 self.run(self.cmd)

619

620 def run(self, cmd):

621 proc = Popen(

622 shlex.split(cmd),

623 stdout=PIPE,

624 stderr=PIPE,

625 stdin=DEVNULL,

626 universal_newlines=True,

627 )

628 self.stdout, self.stderr = proc.communicate()

629 self.returncode = proc.returncode

630

631

632 class LXDTest(object):

633

634 def __init__(self, template=None, rootfs=None):

635 self.rootfs_url = rootfs

636 self.template_url = template

637 self.rootfs_tarball = None

638 self.template_tarball = None

639 self.name = "testbed"

640 self.image_alias = uuid4().hex

641 self.default_remote = "ubuntu:"

642 self.os_version = get_release_to_test()

643

644 def run_command(self, cmd):

645 task = RunCommand(cmd)

646 if task.returncode != 0:

647 logging.error(

648 "Command {} returned a code of {}".format(

649 task.cmd, task.returncode

650 )

651 )

652 logging.error(" STDOUT: {}".format(task.stdout))

653 logging.error(" STDERR: {}".format(task.stderr))

654 return False

655 else:

656 logging.debug("Command {}:".format(task.cmd))

657 if task.stdout != "":

658 logging.debug(" STDOUT: {}".format(task.stdout))

659 elif task.stderr != "":

660 logging.debug(" STDERR: {}".format(task.stderr))

661 else:

662 logging.debug(" Command returned no output")

663 return True

664

665 def setup(self):

666 # Initialize LXD

667 result = True

668 logging.debug("Attempting to initialize LXD")

669 # TODO: Need a method to see if LXD is already initialized

670 if not self.run_command("lxd init --auto"):

671 logging.debug("Error encountered while initializing LXD")

672 result = False

673

674 # Retrieve and insert LXD images

675 if self.template_url is not None:

676 logging.debug("Downloading template.")

677 targetfile = urlparse(self.template_url).path.split("/")[-1]

678 filename = os.path.join("/tmp", targetfile)

679 if not os.path.isfile(filename):

680 self.template_tarball = self.download_images(

681 self.template_url, filename

682 )

683 if not self.template_tarball:

684 logging.error(

685 "Unable to download {} from "

686 "{}".format(self.template_tarball, self.template_url)

687 )

688 logging.error("Aborting")

689 result = False

690 else:

691 logging.debug(

692 "Template file {} already exists. "

693 "Skipping Download.".format(filename)

694 )

695 self.template_tarball = filename

696

697 if self.rootfs_url is not None:

698 logging.debug("Downloading rootfs.")

699 targetfile = urlparse(self.rootfs_url).path.split("/")[-1]

700 filename = os.path.join("/tmp", targetfile)

701 if not os.path.isfile(filename):

702 self.rootfs_tarball = self.download_images(

703 self.rootfs_url, filename

704 )

705 if not self.rootfs_tarball:

706 logging.error(

707 "Unable to download {} from{}".format(

708 self.rootfs_tarball, self.rootfs_url

709 )

710 )

711 logging.error("Aborting")

712 result = False

713 else:

714 logging.debug(

715 "Template file {} already exists. "

716 "Skipping Download.".format(filename)

717 )

718 self.rootfs_tarball = filename

719

720 # Insert images

721 if self.template_url is not None and self.rootfs_url is not None:

722 logging.debug("Importing images into LXD")

723 cmd = "lxc image import {} rootfs {} --alias {}".format(

724 self.template_tarball, self.rootfs_tarball, self.image_alias

725 )

726 result = self.run_command(cmd)

727 if not result:

728 logging.error(

729 "Error encountered while attempting to "

730 "import images into LXD"

731 )

732 result = False

733 else:

734 logging.debug(

735 "No local image available, attempting to "

736 "import from default remote."

737 )

738 retry = 2

739 cmd = "lxc image copy {}{} local: --alias {}".format(

740 self.default_remote, self.os_version, self.image_alias

741 )

742 result = self.run_command(cmd)

743 while not result and retry > 0:

744 logging.error(

745 "Error encountered while attempting to "

746 "import images from default remote."

747 )

748 logging.error("Retrying up to {} times.".format(retry))

749 result = self.run_command(cmd)

750 retry -= 1

751 return result

752

753 def download_images(self, url, filename):

754 """

755 Downloads LXD files for same release as host machine

756 """

757 # TODO: Clean this up to use a non-internet simplestream on MAAS server

758 logging.debug(

759 "Attempting download of {} from {}".format(filename, url)

760 )

761 try:

762 urllib.request.urlretrieve(url, filename)

763 except (

764 IOError,

765 OSError,

766 urllib.error.HTTPError,

767 urllib.error.URLError,

768 ) as exception:

769 logging.error(

770 "Failed download of image from %s: %s", url, exception

771 )

772 return False

773 except ValueError as verr:

774 logging.error("Invalid URL %s" % url)

775 logging.error("%s" % verr)

776 return False

777

778 if not os.path.isfile(filename):

779 logging.warn("Can not find {}".format(filename))

780 return False

781

782 return filename

783

784 def cleanup(self):

785 """

786 Clean up test files an containers created

787 """

788 logging.debug("Cleaning up images and containers created during test")

789 self.run_command("lxc image delete {}".format(self.image_alias))

790 self.run_command("lxc delete --force {}".format(self.name))

791

792 def start(self):

793 """

794 Creates a container and performs the test

795 """

796 result = self.setup()

797 if not result:

798 logging.error("One or more setup stages failed.")

799 return False

800

801 # Create container

802 logging.debug("Launching container")

803 if not self.run_command(

804 "lxc launch {} {}".format(self.image_alias, self.name)

805 ):

806 return False

807

808 logging.debug("Container listing:")

809 cmd = "lxc list"

810 if not self.run_command(cmd):

811 return False

812

813 logging.debug("Testing container")

814 cmd = (

815 "lxc exec {} dd if=/dev/urandom of=testdata.txt "

816 "bs=1024 count=1000".format(self.name)

817 )

818 if not self.run_command(cmd):

819 return False

820

821 return True

822

823

824 class LXDTest_vm(object):

825

826 def __init__(self, template=None, image=None):

827 self.image_url = image

828 self.template_url = template

829 self.image_tarball = None

830 self.template_tarball = None

831 self.name = "testbed"

832 self.image_alias = uuid4().hex

833 self.default_remote = "ubuntu:"

834 self.os_version = get_release_to_test()

835

836 def run_command(self, cmd, log_stderr=True):

837 task = RunCommand(cmd)

838 if task.returncode != 0:

839 logging.error(

840 "Command {} returned a code of {}".format(

841 task.cmd, task.returncode

842 )

843 )

844 logging.error(" STDOUT: {}".format(task.stdout))

845 if log_stderr:

846 logging.error(" STDERR: {}".format(task.stderr))

847 return False

848 else:

849 logging.debug("Command {}:".format(task.cmd))

850 if task.stdout != "":

851 logging.debug(" STDOUT: {}".format(task.stdout))

852 if task.stderr and log_stderr:

853 logging.debug(" STDERR: {}".format(task.stderr))

854 if not (task.stderr or task.stdout): 854 ↛ 855 line 854 didn't jump to line 855 because the condition on line 854 was never true

855 logging.debug(" Command returned no output")

856 return True

857

858 def setup(self):

859 # Initialize LXD

860 result = True

861 logging.debug("Attempting to initialize LXD")

862 # TODO: Need a method to see if LXD is already initialized

863 if not self.run_command("lxd init --auto"): 863 ↛ 868 line 863 didn't jump to line 868 because the condition on line 863 was always true

864 logging.debug("Error encounterd while initializing LXD")

865 result = False

866

867 # Retrieve and insert LXD images

868 if self.template_url is not None: 868 ↛ 869 line 868 didn't jump to line 869 because the condition on line 868 was never true

869 logging.debug("Downloading template.")

870 targetfile = urlparse(self.template_url).path.split("/")[-1]

871 filename = os.path.join("/tmp", targetfile)

872 if not os.path.isfile(filename):

873 self.template_tarball = self.download_images(

874 self.template_url, filename

875 )

876 if not self.template_tarball:

877 logging.error(

878 "Unable to download {} from "

879 "{}".format(self.template_tarball, self.template_url)

880 )

881 logging.error("Aborting")

882 result = False

883 else:

884 logging.debug(

885 "Template file {} already exists. "

886 "Skipping Download.".format(filename)

887 )

888 self.template_tarball = filename

889

890 if self.image_url is not None: 890 ↛ 891 line 890 didn't jump to line 891 because the condition on line 890 was never true

891 logging.debug("Downloading image.")

892 targetfile = urlparse(self.image_url).path.split("/")[-1]

893 filename = os.path.join("/tmp", targetfile)

894 if not os.path.isfile(filename):

895 self.image_tarball = self.download_images(

896 self.image_url, filename

897 )

898 if not self.image_tarball:

899 logging.error(

900 "Unable to download {} from{}".format(

901 self.image_tarball, self.image_url

902 )

903 )

904 logging.error("Aborting")

905 result = False

906 else:

907 logging.debug(

908 "Template file {} already exists. "

909 "Skipping Download.".format(filename)

910 )

911 self.image_tarball = filename

912

913 # Insert images

914 if self.template_url is not None and self.image_url is not None: 914 ↛ 915 line 914 didn't jump to line 915 because the condition on line 914 was never true

915 logging.debug("Importing images into LXD")

916 cmd = "lxc image import {} {} --alias {}".format(

917 self.template_tarball, self.image_tarball, self.image_alias

918 )

919 result = self.run_command(cmd)

920 if not result:

921 logging.error(

922 "Error encountered while attempting to "

923 "import images into LXD"

924 )

925 result = False

926 return result

927

928 def download_images(self, url, filename):

929 """

930 Downloads LXD files for same release as host machine

931 """

932 # TODO: Clean this up to use a non-internet simplestream on MAAS server

933 logging.debug(

934 "Attempting download of {} from {}".format(filename, url)

935 )

936 try:

937 urllib.request.urlretrieve(url, filename)

938 except (

939 IOError,

940 OSError,

941 urllib.error.HTTPError,

942 urllib.error.URLError,

943 ) as exception:

944 logging.error(

945 "Failed download of image from %s: %s", url, exception

946 )

947 return False

948 except ValueError as verr:

949 logging.error("Invalid URL %s" % url)

950 logging.error("%s" % verr)

951 return False

952

953 if not os.path.isfile(filename):

954 logging.warn("Can not find {}".format(filename))

955 return False

956

957 return filename

958

959 def cleanup(self):

960 """

961 Clean up test files an Virtual Machines created

962 """

963 logging.debug("Cleaning up images and VMs created during test")

964 self.run_command("lxc image delete {}".format(self.image_alias), False)

965 self.run_command("lxc delete --force {}".format(self.name), False)

966

967 def start_vm(self):

968 """

969 Creates an lxd virtual machine and performs the test

970 """

971

972 if not self.setup():

973 logging.error("One or more setup stages failed.")

974 return False

975

976 # Create Virtual Machine

977 logging.debug("Launching Virtual Machine")

978 if not self.image_url and not self.template_url:

979 logging.debug(

980 "No local image available, attempting to "

981 "import from default remote."

982 )

983 cmd = "lxc init {}{} {} --vm ".format(

984 self.default_remote, self.os_version, self.name

985 )

986 else:

987 cmd = "lxc init {} {} --vm".format(self.image_alias, self.name)

988

989 if not self.run_command(cmd):

990 return False

991

992 logging.debug("Start VM:")

993 if not self.run_command("lxc start {} ".format(self.name)):

994 return False

995

996 logging.debug("Virtual Machine listing:")

997 if not self.run_command("lxc list"):

998 return False

999

1000 logging.debug("Wait for vm to boot")

1001 wait_interval = 5

1002 max_wait_duration = 300

1003 time_waited = 0

1004 while time_waited < max_wait_duration:

1005 time.sleep(wait_interval)

1006 cmd = "lxc exec {} -- lsb_release -a".format(self.name)

1007 if self.run_command(cmd, False):

1008 print("Vm started and booted successfully")

1009 return True

1010 logging.debug("Re-verify VM booted")

1011 time_waited += wait_interval

1012

1013 logging.debug("testing vm failed")

1014 return False

1015

1016

1017 def test_lxd_vm(args):

1018 logging.debug("Executing LXD VM Test")

1019

1020 template = None

1021 image = None

1022

1023 # First in priority are environment variables.

1024 if "LXD_TEMPLATE" in os.environ:

1025 template = os.environ["LXD_TEMPLATE"]

1026 if "KVM_IMAGE" in os.environ:

1027 image = os.environ["KVM_IMAGE"]

1028

1029 # Finally, highest-priority are command line arguments.

1030 if args.template:

1031 template = args.template

1032 if args.image:

1033 image = args.image

1034

1035 lxd_test = LXDTest_vm(template, image)

1036

1037 result = lxd_test.start_vm()

1038 lxd_test.cleanup()

1039 if result:

1040 print("PASS: Virtual Machine was successfully started and checked")

1041 sys.exit(0)

1042 else:

1043 print("FAIL: Virtual Machine was not started and checked")

1044 sys.exit(1)

1045

1046

1047 def test_lxd(args):

1048 logging.debug("Executing LXD Test")

1049

1050 template = None

1051 rootfs = None

1052

1053 # First in priority are environment variables.

1054 if "LXD_TEMPLATE" in os.environ:

1055 template = os.environ["LXD_TEMPLATE"]

1056 if "LXD_ROOTFS" in os.environ:

1057 rootfs = os.environ["LXD_ROOTFS"]

1058

1059 # Finally, highest-priority are command line arguments.

1060 if args.template:

1061 template = args.template

1062 if args.rootfs:

1063 rootfs = args.rootfs

1064

1065 lxd_test = LXDTest(template, rootfs)

1066

1067 result = lxd_test.start()

1068 lxd_test.cleanup()

1069 if result:

1070 print("PASS: Container was succssfully started and checked")

1071 sys.exit(0)

1072 else:

1073 print("FAIL: Container was not started and checked")

1074 sys.exit(1)

1075

1076

1077 def test_kvm(args):

1078 logging.debug("Executing KVM Test")

1079

1080 image = ""

1081 timeout = ""

1082

1083 # First in priority are environment variables.

1084 if "KVM_TIMEOUT" in os.environ:

1085 try:

1086 timeout = float(os.environ["KVM_TIMEOUT"])

1087 except ValueError as err:

1088 logging.warning("TIMEOUT env variable: %s" % err)

1089 timeout = DEFAULT_TIMEOUT

1090 if "KVM_IMAGE" in os.environ:

1091 image = os.environ["KVM_IMAGE"]

1092

1093 # Finally, highest-priority are command line arguments.

1094 if args.timeout:

1095 timeout = args.timeout

1096 elif not timeout:

1097 timeout = DEFAULT_TIMEOUT

1098 if args.image:

1099 image = args.image

1100

1101 kvm_test = KVMTest(image, timeout, args.log_file)

1102 # If arch is ppc64el, disable smt

1103 if kvm_test.arch == "ppc64el":

1104 os.system("/usr/sbin/ppc64_cpu --smt=off")

1105 result = kvm_test.start()

1106 # If arch is ppc64el, re-enable smt

1107 if kvm_test.arch == "ppc64el":

1108 os.system("/usr/sbin/ppc64_cpu --smt=on")

1109

1110 sys.exit(result)

1111

1112

1113 def main():

1114

1115 parser = ArgumentParser(description="Virtualization Test")

1116 subparsers = parser.add_subparsers()

1117

1118 # Main cli options

1119 kvm_test_parser = subparsers.add_parser(

1120 "kvm", help=("Run kvm virtualization test")

1121 )

1122 lxd_test_parser = subparsers.add_parser(

1123 "lxd", help=("Run the LXD validation test")

1124 )

1125 lxd_test_vm_parser = subparsers.add_parser(

1126 "lxdvm", help=("Run the LXD VM validation test")

1127 )

1128 parser.add_argument(

1129 "--debug",

1130 dest="log_level",

1131 action="store_const",

1132 const=logging.DEBUG,

1133 default=logging.INFO,

1134 )

1135

1136 # Sub test options

1137 kvm_test_parser.add_argument("-i", "--image", type=str, default=None)

1138 kvm_test_parser.add_argument("-t", "--timeout", type=int)

1139 kvm_test_parser.add_argument(

1140 "-l",

1141 "--log-file",

1142 default="virt_debug",

1143 help="Location for debugging output log. Defaults to %(default)s.",

1144 )

1145 kvm_test_parser.set_defaults(func=test_kvm)

1146

1147 # Sub test options

1148 lxd_test_parser.add_argument("--template", type=str, default=None)

1149 lxd_test_parser.add_argument("--rootfs", type=str, default=None)

1150 lxd_test_parser.set_defaults(func=test_lxd)

1151

1152 # Sub test options

1153 lxd_test_vm_parser.add_argument("--template", type=str, default=None)

1154 lxd_test_vm_parser.add_argument("--image", type=str, default=None)

1155 lxd_test_vm_parser.set_defaults(func=test_lxd_vm)

1156

1157 args = parser.parse_args()

1158

1159 try:

1160 logging.basicConfig(level=args.log_level)

1161 except AttributeError:

1162 pass # avoids exception when trying to run without specifying 'kvm'

1163

1164 # silence normal output from requests module

1165 logging.getLogger("requests").setLevel(logging.WARNING)

1166 logging.getLogger("urllib3").setLevel(logging.WARNING)

1167

1168 # Verify args

1169 try:

1170 args.func(args)

1171 except AttributeError:

1172 parser.print_help()

1173 return 1

1174

1175

1176 if __name__ == "__main__":

1177 sys.exit(main())