Debian Codegraph @

rally-jobs/plugins/vpn_utils.py

raw
1 # Copyright 2015 Hewlett-Packard Development Company, L.P.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 # License for the specific language governing permissions and limitations
13 # under the License.
14
15 import os
16 import stat
17 import time
18
19 import paramiko
20 from rally.common import logging
21 from rally.plugins.openstack.wrappers import network as network_wrapper
22 from rally.task import utils as task_utils
23
24 LOG = logging.getLogger(__name__)
25 SUBNET_IP_VERSION = 4
26 START_CIDR = "10.2.0.0/24"
27 EXT_NET_CIDR = "172.16.1.0/24"
28
29
30 def execute_cmd_over_ssh(host, cmd, private_key):
31 """Run the given command over ssh
32
33 Using paramiko package, it creates a connection to the given host;
34 executes the required command on it and returns the output.
35 :param host: Dictionary of ip, username and password
36 :param cmd: Command to be run over ssh
37 :param private_key: path to private key file
38 :return: Output of the executed command
39 """
40 LOG.debug('EXECUTE COMMAND <%s> OVER SSH', cmd)
41 client = paramiko.SSHClient()
42 client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
43 k = paramiko.RSAKey.from_private_key_file(private_key)
44
45 try:
46
47 client.connect(host["ip"], username=host["username"], pkey=k)
48 except paramiko.BadHostKeyException as e:
49 raise Exception(
50 "BADHOSTKEY EXCEPTION WHEN CONNECTING TO %s", host["ip"], e)
51 except paramiko.AuthenticationException as e:
52 raise Exception(
53 "AUTHENTICATION EXCEPTION WHEN CONNECTING TO %s", host["ip"], e)
54 except paramiko.SSHException as e:
55 raise Exception("SSH EXCEPTION WHEN CONNECTING TO %s", host["ip"], e)
56 except OSError as e:
57 raise Exception("SOCKET ERROR WHEN CONNECTING TO %s", host["ip"], e)
58 LOG.debug("CONNECTED TO HOST <%s>", host["ip"])
59 try:
60 stdin, stdout, stderr = client.exec_command(cmd)
61 return stdout.read().splitlines()
62 except paramiko.SSHException as e:
63 raise Exception("SSHEXCEPTION WHEN CONNECTING TO %s", host["ip"], e)
64 finally:
65 client.close()
66
67
68 def create_tenant(keystone_client, tenant_suffix):
69 """Creates keystone tenant with a random name.
70
71 :param keystone_client: keystone client
72 :param tenant_suffix: suffix name for the tenant
73 :returns: uuid of the new tenant
74 """
75 tenant_name = "rally_tenant_" + tenant_suffix
76 LOG.debug("CREATING NEW TENANT %s", tenant_name)
77 return keystone_client.tenants.create(tenant_name).id
78
79
80 def create_network(neutron_client, neutron_admin_client, network_suffix,
81 tenant_id=None, DVR_flag=True, ext_net_name=None):
82 """Create neutron network, subnet, router
83
84 :param neutron_client: neutron client
85 :param neutron_admin_client: neutron client with admin credentials
86 :param network_suffix: str, suffix name of the new network
87 :param tenant_id: uuid of the tenant
88 :param DVR_flag: True - creates a DVR router
89 False - creates a non DVR router
90 :param ext_net_name: external network that is to be used
91 :return: router, subnet, network, subnet_cidr
92 """
93 subnet_cidr = network_wrapper.generate_cidr(start_cidr=START_CIDR)
94
95 def _create_network(neutron_client, network_suffix, is_external=False):
96 """Creates neutron network"""
97
98 network_name = "rally_network_" + network_suffix
99 network_args = {"name": network_name,
100 "router:external": is_external
101 }
102 if tenant_id:
103 network_args["tenant_id"] = tenant_id
104 LOG.debug("ADDING NEW NETWORK %s", network_name)
105 return neutron_client.create_network({"network": network_args})
106
107 def _create_subnet(neutron_client, rally_network, network_suffix, cidr):
108 """Create neutron subnet"""
109
110 network_id = rally_network["network"]["id"]
111 subnet_name = "rally_subnet_" + network_suffix
112 subnet_args = {"name": subnet_name,
113 "cidr": cidr,
114 "network_id": network_id,
115 "ip_version": SUBNET_IP_VERSION
116 }
117 if tenant_id:
118 subnet_args["tenant_id"] = tenant_id
119 LOG.debug("ADDING SUBNET %s", subnet_name)
120 return neutron_client.create_subnet({"subnet": subnet_args})
121
122 def _create_router(neutron_client, ext_network_id, rally_subnet, dvr_flag):
123 """Create router, set the external gateway and add router interface
124
125 :param neutron_client: neutron_client
126 :param ext_network_id: uuid of the external network
127 :param rally_subnet: subnet to add router interface
128 :param dvr_flag: True - creates a DVR router
129 False - creates a non DVR router
130 :return: router
131 """
132 router_name = "rally_router_" + network_suffix
133 gw_info = {"network_id": ext_network_id}
134 router_args = {"name": router_name,
135 "external_gateway_info": gw_info
136 }
137 if not dvr_flag:
138 router_args["distributed"] = dvr_flag
139 if tenant_id:
140 router_args["tenant_id"] = 'tenant_id'
141 LOG.debug("ADDING ROUTER %s", router_name)
142 rally_router = neutron_client.create_router({"router": router_args})
143
144 LOG.debug("[%s]: ADDING ROUTER INTERFACE")
145 neutron_client.add_interface_router(
146 rally_router['router']["id"],
147 {"subnet_id": rally_subnet["subnet"]["id"]})
148 return rally_router
149
150 def _get_external_network_id(ext_net_name):
151 """Fetch the network id for the given external network, if it exists.
152 Else fetch the first external network present.
153 """
154
155 ext_nets = neutron_client.list_networks(
156 **{'router:external': True})['networks']
157
158 ext_nets_searched = [n for n in ext_nets if n['name'] == ext_net_name]
159 if ext_nets_searched:
160 return ext_nets_searched[0]['id']
161 elif ext_nets:
162 return ext_nets[0]['id']
163 else:
164 return None
165
166 def _create_external_network():
167 """Creat external network and subnet"""
168
169 ext_net = _create_network(neutron_admin_client, "public", True)
170 _create_subnet(neutron_admin_client, ext_net, "public", EXT_NET_CIDR)
171 return ext_net['network']['id']
172
173 ext_network_id = _get_external_network_id(ext_net_name)
174 if not ext_network_id:
175 ext_network_id = _create_external_network()
176 rally_network = _create_network(neutron_client, network_suffix)
177 rally_subnet = _create_subnet(neutron_client, rally_network,
178 network_suffix, subnet_cidr)
179 rally_router = _create_router(neutron_client, ext_network_id,
180 rally_subnet, DVR_flag)
181 return rally_router, rally_network, rally_subnet, subnet_cidr
182
183
184 def create_keypair(nova_client, keypair_suffix):
185 """Create keypair
186
187 :param nova_client: nova_client
188 :param keypair_suffix: sufix name for the keypair
189 :return: keypair
190 """
191 keypair_name = "rally_keypair_" + keypair_suffix
192 LOG.debug("CREATING A KEYPAIR %s", keypair_name)
193 keypair = nova_client.keypairs.create(keypair_name)
194 return keypair
195
196
197 def write_key_to_local_path(keypair, local_key_file):
198 """Write the private key of the nova instance to a temp file
199
200 :param keypair: nova keypair
201 :param local_key_file: path to private key file
202 :return:
203 """
204
205 with open(local_key_file, 'w') as f:
206 os.chmod(local_key_file, stat.S_IREAD | stat.S_IWRITE)
207 f.write(keypair.private_key)
208
209
210 def write_key_to_compute_node(keypair, local_path, remote_path, host,
211 private_key):
212 """Write the private key of the nova instance to the compute node
213
214 First fetches the private key from the keypair and writes it to a
215 temporary file in the local machine. It then sftp's the file
216 to the compute host.
217
218 :param keypair: nova keypair
219 :param local_path: path to private key file of the nova instance in the
220 local machine
221 :param remote_path: path where the private key file has to be placed
222 in the remote machine
223 :param host: compute host credentials
224 :param private_key: path to your private key file
225 :return:
226 """
227
228 LOG.debug("WRITING PRIVATE KEY TO COMPUTE NODE")
229 k = paramiko.RSAKey.from_private_key_file(private_key)
230 write_key_to_local_path(keypair, local_path)
231 try:
232 transport = paramiko.Transport(host['ip'], host['port'])
233 except paramiko.SSHException as e:
234 raise Exception(
235 "PARAMIKO TRANSPORT FAILED. CHECK IF THE HOST IP %s AND PORT %s "
236 "ARE CORRECT %s", host['ip'], host['port'], e)
237 try:
238 transport.connect(
239 username=host['username'], pkey=k)
240 except paramiko.BadHostKeyException as e:
241 transport.close()
242 raise Exception(
243 "BADHOSTKEY EXCEPTION WHEN CONNECTING TO %s", host["ip"], e)
244 except paramiko.AuthenticationException as e:
245 transport.close()
246 raise Exception("AUTHENTICATION EXCEPTION WHEN CONNECTING TO %s",
247 host["ip"], e)
248 except paramiko.SSHException as e:
249 transport.close()
250 raise Exception("SSH EXCEPTION WHEN CONNECTING TO %s", host["ip"], e)
251 LOG.debug("CONNECTED TO HOST <%s>", host["ip"])
252
253 try:
254 sftp_client = paramiko.SFTPClient.from_transport(transport)
255 sftp_client.put(local_path, remote_path)
256 except OSError as e:
257 raise Exception("FILE PATH DOESN'T EXIST", e)
258 finally:
259 transport.close()
260
261
262 def create_server(nova_client, keypair, **kwargs):
263 """Create nova instance
264
265 :param nova_client: nova client
266 :param keypair: key-pair to allow ssh
267 :return: new nova instance
268 """
269 # add sec-group
270 sec_group_name = "rally_secgroup_" + kwargs["sec_group_suffix"]
271 LOG.debug("ADDING NEW SECURITY GROUP %s", sec_group_name)
272 secgroup = nova_client.security_groups.create(sec_group_name,
273 sec_group_name)
274 # add security rules for SSH and ICMP
275 nova_client.security_group_rules.create(secgroup.id, from_port=22,
276 to_port=22, ip_protocol="tcp", cidr="0.0.0.0/0")
277
278 nova_client.security_group_rules.create(secgroup.id, from_port=-1,
279 to_port=-1, ip_protocol="icmp", cidr="0.0.0.0/0")
280
281 # boot new nova instance
282 server_name = "rally_server_" + (kwargs["server_suffix"])
283 LOG.debug("BOOTING NEW INSTANCE: %s", server_name)
284 LOG.debug("%s", kwargs["image"])
285 server = nova_client.servers.create(server_name,
286 image=kwargs["image"],
287 flavor=kwargs["flavor"],
288 key_name=keypair.name,
289 security_groups=[secgroup.id],
290 nics=kwargs["nics"])
291 return server
292
293
294 def assert_server_status(server, **kwargs):
295 """Assert server status
296
297 :param server: nova server
298 """
299
300 LOG.debug('WAITING FOR SERVER TO GO ACTIVE')
301 server = task_utils.wait_for(
302 server,
303 is_ready=task_utils.resource_is("ACTIVE"),
304 update_resource=task_utils.get_from_manager(),
305 timeout=kwargs["nova_server_boot_timeout"],
306 check_interval=5)
307 LOG.debug("SERVER STATUS: %s", server.status)
308 assert ('ACTIVE' == server.status), ("THE INSTANCE IS NOT IN ACTIVE STATE")
309
310
311 def get_server_ip(nova_client, server_id, network_suffix):
312 """Get the ip associated with the nova instance
313
314 :param nova_client: nova client
315 :param server_id: uuid of the nova instance whose ip is required
316 :param network_suffix: suffix name of the network
317 :return: ip address of the instance
318 """
319 network_name = "rally_network_" + network_suffix
320 server_details = nova_client.servers.get(server_id)
321 server_ip = server_details.addresses[network_name][0]["addr"]
322 return server_ip
323
324
325 def add_floating_ip(nova_client, server):
326 """Associates floating-ip to a server
327
328 :param nova_client: nova client
329 :param server: nova instance
330 :return: associated floating ip
331 """
332
333 fip_list = nova_client.floating_ips.list()
334 for fip in fip_list:
335 if fip.instance_id is None:
336 floating_ip = fip
337 break
338 else:
339 LOG.debug("CREATING NEW FLOATING IP")
340 floating_ip = nova_client.floating_ips.create()
341 LOG.debug("ASSOCIATING FLOATING IP %s", floating_ip.ip)
342 nova_client.servers.add_floating_ip(server.id, floating_ip.ip)
343 return floating_ip
344
345
346 def get_namespace(host, private_key):
347 """SSH into the host and get the namespaces
348
349 :param host : dictionary of controller/compute node credentials
350 {ip:x.x.x.x, username:xxx, password:xxx}
351 :param private_key: path to private key file
352 :return: namespaces
353 """
354 LOG.debug("GET NAMESPACES")
355 cmd = "sudo ip netns"
356 namespaces = execute_cmd_over_ssh(host, cmd, private_key)
357 LOG.debug("NAMESPACES %s", namespaces)
358 return namespaces
359
360
361 def wait_for_namespace_creation(namespace_tag, router_id, hosts, private_key,
362 timeout=60):
363 """Wait for the namespace creation
364
365 Get into each of the controllers/compute nodes and check which one contains
366 the snat/qrouter namespace corresponding to rally_router. Sleep for a sec
367 and repeat until either the namespace is found or the namespace_creation_
368 time exceeded.
369 :param namespace_tag: which namespace ("snat_" or "qrouter_")
370 :param router_id: uuid of the rally_router
371 :param hosts: controllers or compute hosts
372 :param private_key: path to private key file
373 :param timeout: namespace creation time
374 :return:
375 """
376 start_time = time.time()
377 while True:
378 for host in hosts:
379 namespaces = get_namespace(host, private_key)
380 for line in namespaces:
381 if line == (namespace_tag + router_id):
382 namespace_tag = line
383 return namespace_tag, host
384 time.sleep(1)
385 if time.time() - start_time > timeout:
386 raise Exception("TIMEOUT WHILE WAITING FOR"
387 " NAMESPACES TO BE CREATED")
388
389
390 def ping(host, cmd, private_key):
391 """Execute ping command over ssh"""
392 ping_result = execute_cmd_over_ssh(host, cmd, private_key)
393 if ping_result:
394 LOG.debug("PING RESULT %s", ping_result)
395 return True
396 else:
397 return False
398
399
400 def ping_router_gateway(namespace_controller_tuple, router_gw_ip, private_key):
401 """Ping the ip address from network namespace
402
403 Get into controller's snat-namespaces and ping the peer router gateway ip.
404 :param namespace_controller_tuple: namespace, controller tuple. (It's the
405 controller that contains the namespace )
406 :param router_gw_ip: ip address to be pinged
407 :param private_key: path to private key file
408 :return: True if ping succeeds
409 False if ping fails
410 """
411 namespace, controller = namespace_controller_tuple
412 LOG.debug("PING %s FROM THE NAMESPACE %s", router_gw_ip, namespace)
413 count = 4
414 cmd = "sudo ip netns exec {} ping -w {} -c {} {}".format(
415 namespace, 2 * count, count, router_gw_ip)
416 return ping(controller, cmd, private_key)
417
418
419 def get_interfaces(namespace_controller_tuple, private_key):
420 """Get the interfaces
421
422 Get into the controller's snat namespace and list the interfaces.
423 :param namespace_controller_tuple: namespace, controller tuple(the
424 controller that contains the namespace).
425 :param private_key: path to private key file
426 :return: interfaces
427 """
428 namespace, controller = namespace_controller_tuple
429 LOG.debug("GET THE INTERFACES BY USING 'ip a' FROM THE NAMESPACE %s",
430 namespace)
431 cmd = f"sudo ip netns exec {namespace} ip a"
432 interfaces = execute_cmd_over_ssh(controller, cmd, private_key)
433 LOG.debug("INTERFACES %s", interfaces)
434 return interfaces
435
436
437 def start_tcpdump(namespace_controller_tuple, interface, private_key):
438 """Start the tcpdump at the given interface
439
440 Get into the controller's snat namespace and start a tcp dump at the
441 qg-interface.
442 :param namespace_controller_tuple: namespace, controller tuple. (It's the
443 controller that contains the namespace )
444 :param interface: interface in which tcpdump has to be run
445 :param private_key: path to private key file
446 :return: tcpdump output
447 """
448 namespace, controller = namespace_controller_tuple
449 LOG.debug("START THE TCPDUMP USING 'tcpdump -i %s FROM THE NAMESPACE"
450 " %s", interface, namespace)
451 cmd = ("sudo ip netns exec {} timeout 15 tcpdump -n -i {}"
452 .format(namespace, interface))
453 tcpdump = execute_cmd_over_ssh(controller, cmd, private_key)
454 LOG.debug("TCPDUMP %s", tcpdump)
455 return tcpdump
456
457
458 def ssh_and_ping_server(local_server, peer_server, ns_compute_tuple, keyfile,
459 private_key):
460 """SSH and ping the nova instance from the namespace
461
462 Get into the compute node's qrouter namespace and then ssh into the local
463 nova instance & ping the peer nova instance.
464 :param local_server: private ip of the server to ssh into
465 :param peer_server: private ip of the server to ping to
466 :param ns_compute_tuple: namespace, compute tuple. (It's the
467 compute node that contains the namespace )
468 :param keyfile: path to private key file of the nova instance
469 :param private_key: path to private key file
470 :return: True if ping succeeds
471 False if ping fails
472 """
473 namespace, compute_host = ns_compute_tuple
474 LOG.debug("SSH INTO SERVER %s AND PING THE PEER SERVER %s FROM THE"
475 " NAMESPACE %s", local_server, peer_server, namespace)
476 host = "cirros@" + local_server
477 count = 20
478 cmd = ("sudo ip netns exec {} ssh -v -o StrictHostKeyChecking=no -o"
479 "HashKnownHosts=no -i {} {} ping -w {} -c {} {}"
480 .format(namespace, keyfile, host, 2 * count, count, peer_server))
481 return ping(compute_host, cmd, private_key)
482
483
484 def ssh_and_ping_server_with_fip(local_server, peer_server, keyfile,
485 private_key):
486 """SSH into the local nova instance and ping the peer instance using fips
487
488 :param local_server: fip of the server to ssh into
489 :param peer_server: private ip of the server to ping to
490 :param keyfile: path to private key file of the nova instance
491 :param private_key: path to private key file
492 :return: True if ping succeeds
493 False if ping fails
494 """
495 LOG.debug("SSH INTO LOCAL SERVER %s AND PING THE PEER SERVER %s",
496 local_server.ip, peer_server)
497 count = 20
498 local_host = {"ip": "127.0.0.1", "username": None}
499 host = "cirros@" + local_server.ip
500 cmd = ("ssh -v -o StrictHostKeyChecking=no -o"
501 "HashKnownHosts=no -i {} {} ping -w {} -c {} {}"
502 .format(keyfile, host, 2 * count, count, peer_server))
503 return ping(local_host, cmd, private_key)
504
505
506 def delete_servers(nova_client, servers):
507 """Delete nova servers
508
509 It deletes the nova servers, associated security groups.
510
511 :param nova_client: nova client
512 :param servers: nova instances to be deleted
513 :return:
514 """
515 for server in servers:
516 LOG.debug("DELETING NOVA INSTANCE: %s", server.id)
517 sec_group_id = server.security_groups[0]['name']
518 nova_client.servers.delete(server.id)
519
520 LOG.debug("WAITING FOR INSTANCE TO GET DELETED")
521 task_utils.wait_for_delete(
522 server, update_resource=task_utils.get_from_manager())
523
524 for secgroup in nova_client.security_groups.list():
525 if secgroup.id == sec_group_id:
526 LOG.debug("DELETING SEC_GROUP: %s", sec_group_id)
527 nova_client.security_groups.delete(secgroup.id)
528
529
530 def delete_floating_ips(nova_client, fips):
531 """Delete floating ips
532
533 :param nova_client: nova client
534 :param fips: list of floating ips
535 :return:
536 """
537 for fip in fips:
538 nova_client.floating_ips.delete(fip.id)
539
540
541 def delete_keypairs(nova_client, keypairs):
542 """Delete key pairs
543
544 :param nova_client: nova client
545 :param keypairs: list of keypairs
546 :return
547 """
548 for key_pair in keypairs:
549 LOG.debug("DELETING KEY_PAIR %s", key_pair.name)
550 nova_client.keypairs.delete(key_pair.id)
551
552
553 def delete_networks(neutron_client, neutron_admin_client,
554 routers, networks, subnets):
555 """Delete neutron network, subnets amd routers
556
557 :param neutron_client: neutron client
558 :param neutron_admin_client: neutron_admin_client
559 :param routers: list of routers to be deleted
560 :param networks: list of networks to be deleted
561 :param subnets: list of subnets to be deleted
562 :return
563 """
564 LOG.debug("DELETING RALLY ROUTER INTERFACES & GATEWAYS")
565 for router in routers:
566 neutron_client.remove_gateway_router(router['router']['id'])
567 router_name = router['router']['name']
568 subnet_name = ("rally_subnet_" + router_name[13:len(router_name)])
569 for subnet in subnets:
570 if subnet_name == subnet['subnet']['name']:
571 neutron_client.remove_interface_router(
572 router['router']['id'],
573 {"subnet_id": subnet['subnet']['id']})
574
575 LOG.debug("DELETING RALLY ROUTERS")
576 for router in routers:
577 neutron_client.delete_router(router['router']['id'])
578
579 LOG.debug("DELETING RALLY NETWORKS")
580 for network in networks:
581 if (network['network']['router:external'] and
582 network['network']['name'] == "rally_network_public"):
583 external_network = network
584 neutron_admin_client.delete_network(
585 external_network['network']["id"])
586 elif network['network']['router:external']:
587 pass
588 else:
589 neutron_client.delete_network(network['network']['id'])
590
591
592 def delete_tenants(keystone_client, tenant_ids):
593 """Delete keystone tenant
594
595 :param keystone_client: keystone client
596 :param tenant_ids: list of tenants' uuids
597 :returns: delete keystone tenant instance
598 """
599 LOG.debug('DELETE TENANTS')
600 for id in tenant_ids:
601 keystone_client.tenants.delete(id)
602
603
604 def delete_keyfiles(local_key_files, remote_key_files=None,
605 ns_compute_tuples=None, private_key=None):
606 """Delete the SSH keyfiles from the compute and the local nodes
607
608 :param local_key_files: paths to ssh key files in local node
609 :param remote_key_files: paths to ssh key files in compute nodes
610 :param ns_compute_tuples: namespace, compute tuple. (It's the
611 compute node that contains the namespace )
612 :param private_key: path to private key file
613 :return:
614 """
615 LOG.debug("DELETING RALLY KEY FILES FROM LOCAL MACHINE")
616 for key in local_key_files:
617 if os.path.exists(key):
618 os.remove(key)
619
620 if ns_compute_tuples:
621 LOG.debug("DELETING RALLY KEY FILES FROM COMPUTE HOSTS")
622 for key, ns_comp in zip(remote_key_files, ns_compute_tuples):
623 cmd = f"sudo rm -f {key}"
624 host = ns_comp[1]
625 execute_cmd_over_ssh(host, cmd, private_key)
626
627
628 def delete_hosts_from_knownhosts_file(hosts, ns_compute_tuples=None,
629 private_key=None):
630 """Remove the hosts from the knownhosts file
631
632 :param hosts: host ips to be removed from /root/.ssh/knownhosts
633 :param ns_compute_tuples: namespace, compute tuple. (It's the
634 compute node that contains the namespace )
635 :param private_key: path to private key file
636 :return:
637 """
638 if ns_compute_tuples:
639 LOG.debug("DELETES HOSTS FROM THE KNOWNHOSTS FILE")
640 for host, ns_comp in zip(hosts, ns_compute_tuples):
641 compute_host = ns_comp[1]
642 cmd = ("sudo ssh-keygen -f /root/.ssh/known_hosts -R"
643 " {}".format(host))
644 execute_cmd_over_ssh(compute_host, cmd, private_key)
645 else:
646 for host in hosts:
647 os.system("sudo ssh-keygen -f /root/.ssh/known_hosts -R"
648 " {}".format(host))