From 96e88b00c18bd7dafd662c8b2e74d2d17f0de04d Mon Sep 17 00:00:00 2001 From: Neill Cox Date: Thu, 19 Oct 2023 16:21:52 +1100 Subject: [PATCH] :sparkles: finish basic functionality of os_migrate_setup.py --- src/tripleo_aio_helpers/os_migrate_setup.py | 433 ++++++++++++++++++-- 1 file changed, 389 insertions(+), 44 deletions(-) diff --git a/src/tripleo_aio_helpers/os_migrate_setup.py b/src/tripleo_aio_helpers/os_migrate_setup.py index 0ed3357..e085830 100644 --- a/src/tripleo_aio_helpers/os_migrate_setup.py +++ b/src/tripleo_aio_helpers/os_migrate_setup.py @@ -3,8 +3,22 @@ Quick and dirty script to help setup project, flavors, networks, images """ import argparse import json +import subprocess +import tempfile + +from .utils import get_from_env, openstack_cmd, test_user_openstack_cmd + + +def execute_cmd(cmd): + """Execute a command""" + return subprocess.check_output(cmd, shell=True, universal_newlines=True) + + +def execute_ssh_cmd(cmd, args): + """Execute a command on a remote host using ssh""" + cmd = f'ssh {args.ssh} "{cmd}"' + return subprocess.check_output(cmd, shell=True, universal_newlines=True) -from .utils import get_from_env,openstack_cmd def parse_args(): """Parse the command line arguments""" @@ -20,12 +34,28 @@ def parse_args(): parser.add_argument("-c", "--cloud", default="standalone") parser.add_argument("-g", "--gateway") parser.add_argument("-C", "--public-network-cidr") + parser.add_argument( + "--ssh-key", help="File containing a public key to inject into the instances" + ) parser.add_argument("--private-network-cidr", default="192.168.100.0/24") parser.add_argument("--public-net-start") parser.add_argument("--public-net-end") parser.add_argument("--dns-server") parser.add_argument("--dry-run", action="store_true") - parser.add_argument("--ssh", help="Connection string to run commands on a remote host.") + parser.add_argument( + "--ssh", help="Connection string to run commands on a remote host." + ) + parser.add_argument( + "--cirros-url", + help="a URL that a cirros image can be downloaded from. Required. " + "Can be set in AIO_CIRROS_URL", + ) + parser.add_argument( + "--rhel-url", + help="a URL that a RHEL image can be downloaded from. Required. Can " + "be set in AIO_RHEL_URL", + ) + parser.add_argument("--debug", action="store_true") # export OS_CLOUD=standalone # export STANDALONE_HOST=10.76.23.39 @@ -34,12 +64,16 @@ def parse_args(): if not args.gateway: args.gateway = get_from_env("--gateway", "AIO_GATEWAY") - + if not args.public_network_cidr: - args.public_network_cidr = get_from_env("--public-network-cidr", "AIO_PUBLIC_CIDR") - + args.public_network_cidr = get_from_env( + "--public-network-cidr", "AIO_PUBLIC_CIDR" + ) + if not args.public_net_start: - args.public_net_start = get_from_env("--public-net-start", "AIO_PUBLIC_NET_START") + args.public_net_start = get_from_env( + "--public-net-start", "AIO_PUBLIC_NET_START" + ) if not args.public_net_end: args.public_net_end = get_from_env("--public-net-end", "AIO_PUBLIC_NET_END") @@ -47,15 +81,24 @@ def parse_args(): if not args.dns_server: args.dns_server = get_from_env("--dns-server", "AIO_DNS_SERVER") + if not args.cirros_url: + args.dns_server = get_from_env("--dns-server", "AIO_CIRROS_URL") + + if not args.rhel_url: + args.dns_server = get_from_env("--dns-server", "AIO_RHEL_URL") + if not args.ssh: args.ssh = get_from_env("--ssh", "AIO_SSH", required=False) + if not args.ssh_key: + args.ssh_key = get_from_env("--ssh-key", "AIO_SSH_KEY") + return args + def create_project(args): """Create the project if it doesn't already exist""" - cmd = "project list -f json" project_exists = [ x @@ -84,7 +127,7 @@ def create_user(args): if args.dry_run: print("Dry run specified. Not creating user") return - + cmd = "user list -f json" user_exists = [ x for x in json.loads(openstack_cmd(cmd, args)) if x["Name"] == args.username @@ -116,7 +159,7 @@ def assign_member_role(args): if args.dry_run: print("Dry run specified. Not assigning role") return - + cmd = f"role add --user {args.username} --project {args.project_id} member" result = openstack_cmd(cmd, args) @@ -129,66 +172,362 @@ def assign_member_role(args): def create_public_network(args): - """Coming soon - create the public network""" - # pylint: disable=unused-argument,unused-variable - print("creating public network - NYI") - cmd = ( - "network create --external --provider-physical-network datacentre " - "--provider-network-type flat public" + """Create the public network""" + + network_exists = json.loads( + openstack_cmd("network list -f json --name public", args) ) - cmd = ( - f"subnet create public-net --subnet-range {args.public_network_cidr} " - f"--no-dhcp --gateway {args.gateway} --allocation-pool " - f"start={args.public_net_start},end={args.public_net_end} " - "--network public" + + if network_exists: + print("Public network already exists - skipping") + args.public_network_id = network_exists[0]["ID"] + else: + cmd = ( + "network create -f json --external --provider-physical-network datacentre " + "--provider-network-type flat public" + ) + try: + args.public_network_id = json.loads(openstack_cmd(cmd, args))["id"] + except json.decoder.JSONDecodeError: + print(cmd) + raise + print("Public network created.") + + subnet_exists = json.loads( + openstack_cmd("subnet list -f json --name public-net", args) ) + if subnet_exists: + print("Public subnet exists - skipping") + else: + cmd = ( + "subnet create public-net -f json " + f"--subnet-range {args.public_network_cidr} " + f"--gateway {args.gateway} " + f"--allocation-pool start={args.public_net_start},end={args.public_net_end} " + "--network public " + f"--host-route destination=0.0.0.0/0,gateway={args.gateway} " + f"--dns-nameserver {args.dns_server}" + ) + args.public_subnet_id = json.loads(openstack_cmd(cmd, args)) + print("Public subnet created.") def create_private_network(args): - """Coming soon - create the private network""" - # pylint: disable=unused-argument,unused-variable - cmd = "openstack network create --internal private" - cmd = ( - "openstack subnet create private-net " - f"--subnet-range {args.private_network_cidr} --network private" + """Create the private network and subnet""" + + network_exists = json.loads( + openstack_cmd( + f"network list -f json --project {args.project_id} --name private", args + ) ) - print("creating private network - NYI") + + if network_exists: + print("Private network already exists - skipping") + args.private_network_id = network_exists[0]["ID"] + else: + cmd = f"network create -f json --internal private --project {args.project_id}" + args.private_network_id = json.loads(openstack_cmd(cmd, args))["id"] + print("Private network created.") + + subnet_exists = json.loads( + openstack_cmd( + f"subnet list -f json --project {args.project_id} --name private-net", args + ) + ) + if subnet_exists: + print("Private subnet exists - skipping") + args.private_subnet_id = subnet_exists[0]["ID"] + else: + cmd = ( + f"subnet create private-net -f json --project {args.project_id} " + f"--subnet-range {args.private_network_cidr} --network {args.private_network_id}" + ) + args.private_subnet_id = json.loads(openstack_cmd(cmd, args)) + print("Private subnet created.") + + +def create_flavor(args, flavor_name, memory, disk, cpus): + """Create a flavor - DRY""" + cmd = "flavor list -f json --all" + # Note we are going to assume that there is only one cirros flavor. This + # will be more of a problem come deletion time. + result = json.loads(openstack_cmd(cmd, args)) + flavor_exists = [x for x in result if x["Name"] == flavor_name] + if flavor_exists: + print(f"{flavor_name} flavor already exists. Skipping creation") + args.__dict__[flavor_name + "_flavor"] = flavor_exists[0]["ID"] + return + + print("creating cirros flavor") + # Note can't add a description in RHOSP16, but perhaps should add for 17 + cmd = f""" + flavor create -f json + --ram {memory} + --disk {disk} + --vcpus {cpus} + --private + --project {args.project_id} + {flavor_name} + """.replace( + "\n", " " + ) + + result = openstack_cmd(cmd, args, as_json=True) + args.__dict__[flavor_name] = result["id"] + print(result) def create_cirros_flavor(args): - """Coming soon - create the cirros flavor""" - # pylint: disable=unused-argument - print("creating cirros flavor - NYI") + """create the cirros flavor""" + create_flavor(args, "cirros", 256, 20, 1) def create_rhel_flavor(args): - """Coming soon - create the rhel flavor""" - # pylint: disable=unused-argument - print("creating rhel flavor - NYI") + """create the rhel flavor""" + create_flavor(args, "rhel", 1536, 120, 2) + + +def create_image(image_name, image_url, disk_size, memory, args): + """Create an image - DRY""" + + cmd = "image list -f json" + image_exists = [ + image + for image in json.loads(openstack_cmd(cmd, args)) + if image["Name"] == image_name + ] + + if image_exists: + print(f"{image_name} already exists - not creating.") + args.__dict__[image_name] = image_exists[0]["ID"] + return + + with tempfile.TemporaryDirectory() as tmp_dir: + + # download image to tmpdir + fname = f"{tmp_dir}/{image_name}.img" + cmd = f"wget -O {fname} {image_url}" + _ = execute_cmd(cmd) + + if args.ssh: + # Copy img to remote host + cmd = f"scp {fname} {args.ssh}:{image_name}.img" + execute_cmd(cmd) + + # create image - note this will only work on a remote ssh host at the moment + cmd = f""" + image create -f json + --container-format bare + --disk-format qcow2 + --min-disk {disk_size} --min-ram {memory} + --file {image_name}.img + --private + --project {args.project_id} + {image_name} + """ + + result = openstack_cmd(cmd, args, as_json=True) + args.__dict__[image_name] = result["id"] + + if args.ssh: + # Delete image from remote host + cmd = f"rm {image_name}.img" + execute_ssh_cmd(cmd, args) def create_cirros_image(args): - """Coming soon - create the cirros image""" - # pylint: disable=unused-argument - print("creating cirros image - NYI") + """create the cirros image""" + + create_image("cirros_image", args.cirros_url, 20, 256, args) def create_rhel_image(args): - """Coming soon - create the rhel image""" - # pylint: disable=unused-argument - print("creating rhel image - NYI") + """create the rhel image""" + create_image("rhel_image", args.rhel_url, 120, 1536, args) + + +def create_instance(args, name, flavor, image, security_group, boot_size): + """Create an instance""" + # pylint:disable=too-many-arguments + instance_exists = [ + instance + for instance in test_user_openstack_cmd( + "server list -f json", args, as_json=True + ) + if instance["Name"] == name + ] + + if instance_exists: + print(f"{name} instance exists - skipping") + else: + cmd = ( + f"server create --flavor {flavor} " + f"--image {image} " + f"--key-name test_keypair " + f"--security-group {security_group} " + "--network private " + f"--boot-from-volume {boot_size} " + "-f json " + f"{name}" + ) + + server = test_user_openstack_cmd(cmd, args, as_json=True) + + print(f"{name} instance created") + # assign floating IP + fip = test_user_openstack_cmd( + "floating ip create -f json public", args, as_json=True + ) + _ = test_user_openstack_cmd( + f"server add floating ip {server['id']} {fip['floating_ip_address']}", args + ) def create_cirros_instance(args): - """Coming soon - create the cirros instance""" - # pylint: disable=unused-argument - print("creating cirros instance - NYI") + """Create the cirros instance""" + + create_instance( + args, + "cirros-test-instance", + args.cirros_flavor, + args.cirros_image, + args.sg_id, + 20, + ) def create_rhel_instance(args): """Coming soon - create the rhel instance""" - # pylint: disable=unused-argument - print("creating rhel instance - NYI") + create_instance( + args, + "rhel-test-instance", + args.rhel_flavor, + args.rhel_image, + args.sg_id, + 120 + ) + + +def create_conversion_instance(args): + """Coming soon - create the rhel instance""" + create_instance( + args, + "rhel-conversion", + args.rhel_flavor, + args.rhel_image, + args.sg_id, + 120 + ) + + +def create_keypair(args): + """Create a keypair to allow ssh later""" + + key_exists = [ + kp + for kp in json.loads(test_user_openstack_cmd("keypair list -f json ", args)) + if kp["Name"] == "test_keypair" + ] + + if key_exists: + args.keypair_name = key_exists[0]["Name"] + else: + if args.ssh: + fname = "temp_pub_key" + cmd = f"scp {args.ssh_key} {args.ssh}:{fname}" + + execute_cmd(cmd) + + args.keypair_name = json.loads( + test_user_openstack_cmd( + f"keypair create -f json --public-key {fname} test_keypair", args + ) + )[ + "name" + ] # a little inconsistent here... + + _ = execute_ssh_cmd(f"rm {fname}", args) + else: + raise NotImplementedError("Only works over ssh") + + +def create_router(args): + """Create a router""" + router_id = None + + try: + router_id = [ + router + for router in openstack_cmd("router list -f json", args, as_json=True) + if router["Name"] == "test-router" + ][0]["ID"] + except IndexError: + pass + + if not router_id: + router_id = test_user_openstack_cmd( + "router create test-router -f json", args, as_json=True + )["id"] + print("router created") + + router_info = openstack_cmd(f"router show -f json {router_id}", args, as_json=True) + + if router_info["external_gateway_info"]: + print("router gateway already set") + else: + openstack_cmd( + f"router set {router_id} --external-gateway {args.public_network_id}", args + ) + print("router gateway added") + + if [ + interface + for interface in router_info["interfaces_info"] + if interface["subnet_id"] == args.private_subnet_id + ]: + print("router already connected to private subnet") + else: + cmd = f"router add subnet {router_id} {args.private_subnet_id}" + print(cmd) + openstack_cmd(cmd, args) + print("router subnet added") + + args.router_id = router_id + + +def create_security_group(args): + """Create a security group that allows ssh and icmp""" + + sg_exists = [ + sg + for sg in json.loads( + test_user_openstack_cmd("security group list -f json", args) + ) + if sg["Name"] == "test-sg" + ] + + if sg_exists: + args.sg_id = sg_exists[0]["ID"] + else: + sec_grp = json.loads( + test_user_openstack_cmd( + "security group create test-sg -f json", + args + ) + ) + args.sg_id = sec_grp["id"] + + _ = test_user_openstack_cmd( + ( + "security group rule create --dst-port 22 " + f"--protocol tcp {args.sg_id}" + ), + args, + ) + + _ = test_user_openstack_cmd( + ("security group rule create --protocol icmp " f"{args.sg_id}"), args + ) def main(): @@ -202,6 +541,7 @@ def main(): create_public_network(args) create_private_network(args) + create_router(args) create_cirros_flavor(args) create_rhel_flavor(args) @@ -209,9 +549,14 @@ def main(): create_cirros_image(args) create_rhel_image(args) + create_keypair(args) + create_security_group(args) + create_cirros_instance(args) create_rhel_instance(args) + ## create_os_migrate host + if __name__ == "__main__": main()