✨ finish basic functionality of os_migrate_setup.py

src/tripleo_aio_helpers/os_migrate_setup.py

@@ -3,8 +3,22 @@ Quick and dirty script to help setup project, flavors, networks, images
 """
 import argparse
 import json
+import subprocess
+import tempfile
+
+from .utils import get_from_env, openstack_cmd, test_user_openstack_cmd
+
+
+def execute_cmd(cmd):
+    """Execute a command"""
+    return subprocess.check_output(cmd, shell=True, universal_newlines=True)
+
+
+def execute_ssh_cmd(cmd, args):
+    """Execute a command on a remote host using ssh"""
+    cmd = f'ssh {args.ssh} "{cmd}"'
+    return subprocess.check_output(cmd, shell=True, universal_newlines=True)
 
-from .utils import get_from_env,openstack_cmd
 
 def parse_args():
     """Parse the command line arguments"""
@@ -20,12 +34,28 @@ def parse_args():
     parser.add_argument("-c", "--cloud", default="standalone")
     parser.add_argument("-g", "--gateway")
     parser.add_argument("-C", "--public-network-cidr")
+    parser.add_argument(
+        "--ssh-key", help="File containing a public key to inject into the instances"
+    )
     parser.add_argument("--private-network-cidr", default="192.168.100.0/24")
     parser.add_argument("--public-net-start")
     parser.add_argument("--public-net-end")
     parser.add_argument("--dns-server")
     parser.add_argument("--dry-run", action="store_true")
-    parser.add_argument("--ssh", help="Connection string to run commands on a remote host.")
+    parser.add_argument(
+        "--ssh", help="Connection string to run commands on a remote host."
+    )
+    parser.add_argument(
+        "--cirros-url",
+        help="a URL that a cirros image can be downloaded from. Required. "
+        "Can be set in AIO_CIRROS_URL",
+    )
+    parser.add_argument(
+        "--rhel-url",
+        help="a URL that a RHEL image can be downloaded from. Required. Can "
+        "be set in AIO_RHEL_URL",
+    )
+    parser.add_argument("--debug", action="store_true")
 
     # export OS_CLOUD=standalone
     # export STANDALONE_HOST=10.76.23.39
@@ -34,12 +64,16 @@ def parse_args():
 
     if not args.gateway:
         args.gateway = get_from_env("--gateway", "AIO_GATEWAY")
-    
+
     if not args.public_network_cidr:
-        args.public_network_cidr = get_from_env("--public-network-cidr", "AIO_PUBLIC_CIDR")
+        args.public_network_cidr = get_from_env(
-    
+            "--public-network-cidr", "AIO_PUBLIC_CIDR"
+        )
+
     if not args.public_net_start:
-        args.public_net_start = get_from_env("--public-net-start", "AIO_PUBLIC_NET_START")
+        args.public_net_start = get_from_env(
+            "--public-net-start", "AIO_PUBLIC_NET_START"
+        )
 
     if not args.public_net_end:
         args.public_net_end = get_from_env("--public-net-end", "AIO_PUBLIC_NET_END")
@@ -47,15 +81,24 @@ def parse_args():
     if not args.dns_server:
         args.dns_server = get_from_env("--dns-server", "AIO_DNS_SERVER")
 
+    if not args.cirros_url:
+        args.dns_server = get_from_env("--dns-server", "AIO_CIRROS_URL")
+
+    if not args.rhel_url:
+        args.dns_server = get_from_env("--dns-server", "AIO_RHEL_URL")
+
     if not args.ssh:
         args.ssh = get_from_env("--ssh", "AIO_SSH", required=False)
 
+    if not args.ssh_key:
+        args.ssh_key = get_from_env("--ssh-key", "AIO_SSH_KEY")
+
     return args
 
+
 def create_project(args):
     """Create the project if it doesn't already exist"""
 
-    
     cmd = "project list -f json"
     project_exists = [
         x
@@ -84,7 +127,7 @@ def create_user(args):
     if args.dry_run:
         print("Dry run specified. Not creating user")
         return
-    
+
     cmd = "user list -f json"
     user_exists = [
         x for x in json.loads(openstack_cmd(cmd, args)) if x["Name"] == args.username
@@ -116,7 +159,7 @@ def assign_member_role(args):
     if args.dry_run:
         print("Dry run specified. Not assigning role")
         return
-    
+
     cmd = f"role add --user {args.username} --project {args.project_id} member"
 
     result = openstack_cmd(cmd, args)
@@ -129,66 +172,362 @@ def assign_member_role(args):
 
 
 def create_public_network(args):
-    """Coming soon - create the public network"""
+    """Create the public network"""
-    # pylint: disable=unused-argument,unused-variable
+
-    print("creating public network - NYI")
+    network_exists = json.loads(
-    cmd = (
+        openstack_cmd("network list -f json --name public", args)
-        "network create --external --provider-physical-network datacentre "
-        "--provider-network-type flat public"
     )
-    cmd = (
+
-        f"subnet create public-net --subnet-range {args.public_network_cidr} "
+    if network_exists:
-        f"--no-dhcp --gateway {args.gateway} --allocation-pool "
+        print("Public network already exists - skipping")
-        f"start={args.public_net_start},end={args.public_net_end} "
+        args.public_network_id = network_exists[0]["ID"]
-        "--network public"
+    else:
+        cmd = (
+            "network create -f json  --external --provider-physical-network datacentre "
+            "--provider-network-type flat public"
+        )
+        try:
+            args.public_network_id = json.loads(openstack_cmd(cmd, args))["id"]
+        except json.decoder.JSONDecodeError:
+            print(cmd)
+            raise
+        print("Public network created.")
+
+    subnet_exists = json.loads(
+        openstack_cmd("subnet list -f json --name public-net", args)
     )
+    if subnet_exists:
+        print("Public subnet exists - skipping")
+    else:
+        cmd = (
+            "subnet create public-net -f json "
+            f"--subnet-range {args.public_network_cidr} "
+            f"--gateway {args.gateway} "
+            f"--allocation-pool start={args.public_net_start},end={args.public_net_end} "
+            "--network public "
+            f"--host-route destination=0.0.0.0/0,gateway={args.gateway} "
+            f"--dns-nameserver {args.dns_server}"
+        )
+        args.public_subnet_id = json.loads(openstack_cmd(cmd, args))
+        print("Public subnet created.")
 
 
 def create_private_network(args):
-    """Coming soon - create the private network"""
+    """Create the private network and subnet"""
-    # pylint: disable=unused-argument,unused-variable
+
-    cmd = "openstack network create --internal private"
+    network_exists = json.loads(
-    cmd = (
+        openstack_cmd(
-        "openstack subnet create private-net "
+            f"network list -f json --project {args.project_id} --name private", args
-        f"--subnet-range {args.private_network_cidr} --network private"
+        )
     )
-    print("creating private network - NYI")
+
+    if network_exists:
+        print("Private network already exists - skipping")
+        args.private_network_id = network_exists[0]["ID"]
+    else:
+        cmd = f"network create -f json --internal private --project {args.project_id}"
+        args.private_network_id = json.loads(openstack_cmd(cmd, args))["id"]
+        print("Private network created.")
+
+    subnet_exists = json.loads(
+        openstack_cmd(
+            f"subnet list -f json --project {args.project_id} --name private-net", args
+        )
+    )
+    if subnet_exists:
+        print("Private subnet exists - skipping")
+        args.private_subnet_id = subnet_exists[0]["ID"]
+    else:
+        cmd = (
+            f"subnet create private-net -f json --project {args.project_id} "
+            f"--subnet-range {args.private_network_cidr} --network {args.private_network_id}"
+        )
+        args.private_subnet_id = json.loads(openstack_cmd(cmd, args))
+        print("Private subnet created.")
+
+
+def create_flavor(args, flavor_name, memory, disk, cpus):
+    """Create a flavor - DRY"""
+    cmd = "flavor list -f json --all"
+    # Note we are going to assume that there is only one cirros flavor. This
+    # will be more of a problem come deletion time.
+    result = json.loads(openstack_cmd(cmd, args))
+    flavor_exists = [x for x in result if x["Name"] == flavor_name]
+    if flavor_exists:
+        print(f"{flavor_name} flavor already exists. Skipping creation")
+        args.__dict__[flavor_name + "_flavor"] = flavor_exists[0]["ID"]
+        return
+
+    print("creating cirros flavor")
+    # Note can't add a description in RHOSP16, but perhaps should add for 17
+    cmd = f"""
+        flavor create -f json 
+            --ram {memory}
+            --disk {disk}
+            --vcpus {cpus} 
+            --private 
+            --project {args.project_id} 
+            {flavor_name}
+        """.replace(
+        "\n", " "
+    )
+
+    result = openstack_cmd(cmd, args, as_json=True)
+    args.__dict__[flavor_name] = result["id"]
+    print(result)
 
 
 def create_cirros_flavor(args):
-    """Coming soon - create the cirros flavor"""
+    """create the cirros flavor"""
-    # pylint: disable=unused-argument
+    create_flavor(args, "cirros", 256, 20, 1)
-    print("creating cirros flavor - NYI")
 
 
 def create_rhel_flavor(args):
-    """Coming soon - create the rhel flavor"""
+    """create the rhel flavor"""
-    # pylint: disable=unused-argument
+    create_flavor(args, "rhel", 1536, 120, 2)
-    print("creating rhel flavor - NYI")
+
+
+def create_image(image_name, image_url, disk_size, memory, args):
+    """Create an image - DRY"""
+
+    cmd = "image list -f json"
+    image_exists = [
+        image
+        for image in json.loads(openstack_cmd(cmd, args))
+        if image["Name"] == image_name
+    ]
+
+    if image_exists:
+        print(f"{image_name} already exists - not creating.")
+        args.__dict__[image_name] = image_exists[0]["ID"]
+        return
+
+    with tempfile.TemporaryDirectory() as tmp_dir:
+
+        # download image to tmpdir
+        fname = f"{tmp_dir}/{image_name}.img"
+        cmd = f"wget -O {fname} {image_url}"
+        _ = execute_cmd(cmd)
+
+        if args.ssh:
+            # Copy img to remote host
+            cmd = f"scp {fname} {args.ssh}:{image_name}.img"
+            execute_cmd(cmd)
+
+        # create image - note this will only work on a remote ssh host at the moment
+        cmd = f"""
+            image create -f json
+                --container-format bare
+                --disk-format qcow2
+                --min-disk {disk_size} --min-ram {memory}
+                --file {image_name}.img
+                --private
+                --project {args.project_id}
+                {image_name}
+            """
+
+        result = openstack_cmd(cmd, args, as_json=True)
+        args.__dict__[image_name] = result["id"]
+
+        if args.ssh:
+            # Delete image from remote host
+            cmd = f"rm {image_name}.img"
+            execute_ssh_cmd(cmd, args)
 
 
 def create_cirros_image(args):
-    """Coming soon - create the cirros image"""
+    """create the cirros image"""
-    # pylint: disable=unused-argument
+
-    print("creating cirros image - NYI")
+    create_image("cirros_image", args.cirros_url, 20, 256, args)
 
 
 def create_rhel_image(args):
-    """Coming soon - create the rhel image"""
+    """create the rhel image"""
-    # pylint: disable=unused-argument
+    create_image("rhel_image", args.rhel_url, 120, 1536, args)
-    print("creating rhel image - NYI")
+
+
+def create_instance(args, name, flavor, image, security_group, boot_size):
+    """Create an instance"""
+    # pylint:disable=too-many-arguments
+    instance_exists = [
+        instance
+        for instance in test_user_openstack_cmd(
+            "server list -f json", args, as_json=True
+        )
+        if instance["Name"] == name
+    ]
+
+    if instance_exists:
+        print(f"{name} instance exists - skipping")
+    else:
+        cmd = (
+            f"server create --flavor {flavor} "
+            f"--image {image} "
+            f"--key-name test_keypair "
+            f"--security-group {security_group} "
+            "--network private "
+            f"--boot-from-volume {boot_size} "
+            "-f json "
+            f"{name}"
+        )
+
+        server = test_user_openstack_cmd(cmd, args, as_json=True)
+
+        print(f"{name} instance created")
+        # assign floating IP
+        fip = test_user_openstack_cmd(
+            "floating ip create -f json public", args, as_json=True
+        )
+        _ = test_user_openstack_cmd(
+            f"server add floating ip {server['id']} {fip['floating_ip_address']}", args
+        )
 
 
 def create_cirros_instance(args):
-    """Coming soon - create the cirros instance"""
+    """Create the cirros instance"""
-    # pylint: disable=unused-argument
+
-    print("creating cirros instance - NYI")
+    create_instance(
+        args,
+        "cirros-test-instance",
+        args.cirros_flavor,
+        args.cirros_image,
+        args.sg_id,
+        20,
+    )
 
 
 def create_rhel_instance(args):
     """Coming soon - create the rhel instance"""
-    # pylint: disable=unused-argument
+    create_instance(
-    print("creating rhel instance - NYI")
+        args,
+        "rhel-test-instance",
+        args.rhel_flavor,
+        args.rhel_image,
+        args.sg_id,
+        120
+    )
+
+
+def create_conversion_instance(args):
+    """Coming soon - create the rhel instance"""
+    create_instance(
+        args,
+        "rhel-conversion",
+        args.rhel_flavor,
+        args.rhel_image,
+        args.sg_id,
+        120
+    )
+
+
+def create_keypair(args):
+    """Create a keypair to allow ssh later"""
+
+    key_exists = [
+        kp
+        for kp in json.loads(test_user_openstack_cmd("keypair list -f json ", args))
+        if kp["Name"] == "test_keypair"
+    ]
+
+    if key_exists:
+        args.keypair_name = key_exists[0]["Name"]
+    else:
+        if args.ssh:
+            fname = "temp_pub_key"
+            cmd = f"scp {args.ssh_key} {args.ssh}:{fname}"
+
+            execute_cmd(cmd)
+
+            args.keypair_name = json.loads(
+                test_user_openstack_cmd(
+                    f"keypair create -f json --public-key {fname} test_keypair", args
+                )
+            )[
+                "name"
+            ]  # a little inconsistent here...
+
+            _ = execute_ssh_cmd(f"rm {fname}", args)
+        else:
+            raise NotImplementedError("Only works over ssh")
+
+
+def create_router(args):
+    """Create a router"""
+    router_id = None
+
+    try:
+        router_id = [
+            router
+            for router in openstack_cmd("router list -f json", args, as_json=True)
+            if router["Name"] == "test-router"
+        ][0]["ID"]
+    except IndexError:
+        pass
+
+    if not router_id:
+        router_id = test_user_openstack_cmd(
+            "router create test-router -f json", args, as_json=True
+        )["id"]
+        print("router created")
+
+    router_info = openstack_cmd(f"router show -f json {router_id}", args, as_json=True)
+
+    if router_info["external_gateway_info"]:
+        print("router gateway already set")
+    else:
+        openstack_cmd(
+            f"router set {router_id} --external-gateway {args.public_network_id}", args
+        )
+        print("router gateway added")
+
+    if [
+        interface
+        for interface in router_info["interfaces_info"]
+        if interface["subnet_id"] == args.private_subnet_id
+    ]:
+        print("router already connected to private subnet")
+    else:
+        cmd = f"router add subnet {router_id} {args.private_subnet_id}"
+        print(cmd)
+        openstack_cmd(cmd, args)
+        print("router subnet added")
+
+    args.router_id = router_id
+
+
+def create_security_group(args):
+    """Create a security group that allows ssh and icmp"""
+
+    sg_exists = [
+        sg
+        for sg in json.loads(
+            test_user_openstack_cmd("security group list -f json", args)
+        )
+        if sg["Name"] == "test-sg"
+    ]
+
+    if sg_exists:
+        args.sg_id = sg_exists[0]["ID"]
+    else:
+        sec_grp = json.loads(
+            test_user_openstack_cmd(
+                "security group create test-sg -f json",
+                args
+                )
+        )
+        args.sg_id = sec_grp["id"]
+
+        _ = test_user_openstack_cmd(
+            (
+                "security group rule create --dst-port 22 "
+                f"--protocol tcp {args.sg_id}"
+            ),
+            args,
+        )
+
+        _ = test_user_openstack_cmd(
+            ("security group rule create --protocol icmp " f"{args.sg_id}"), args
+        )
 
 
 def main():
@@ -202,6 +541,7 @@ def main():
 
     create_public_network(args)
     create_private_network(args)
+    create_router(args)
 
     create_cirros_flavor(args)
     create_rhel_flavor(args)
@@ -209,9 +549,14 @@ def main():
     create_cirros_image(args)
     create_rhel_image(args)
 
+    create_keypair(args)
+    create_security_group(args)
+
     create_cirros_instance(args)
     create_rhel_instance(args)
 
+    ## create_os_migrate host
+
 
 if __name__ == "__main__":
     main()