我们从Python开源项目中，提取了以下46个代码示例，用于说明如何使用six.moves.shlex_quote()。
def print_env_vars(credentials, target_profile): aws_access_key_id = shlex_quote(credentials.get( target_profile, 'aws_access_key_id')) aws_secret_access_key = shlex_quote(credentials.get( target_profile, 'aws_secret_access_key')) aws_session_token = shlex_quote(credentials.get( target_profile, 'aws_session_token')) print("AWS_ACCESS_KEY_ID=%s; export AWS_ACCESS_KEY_ID;" % shlex_quote(aws_access_key_id)) print("AWS_SECRET_ACCESS_KEY=%s; export AWS_SECRET_ACCESS_KEY;" % shlex_quote(aws_secret_access_key)) print("AWS_SESSION_TOKEN=%s; export AWS_SESSION_TOKEN;" % shlex_quote(aws_session_token)) # for backwards compatibility with older Boto print("AWS_SECURITY_TOKEN=%s; export AWS_SECURITY_TOKEN;" % shlex_quote(aws_session_token))
def test_export_path(dummy_plugin_dir, register_dummy_plugin, run_command): """ export_path should produce an export statement with composed plugin path """ register_dummy_plugin() os.environ['PYANG_PLUGINPATH'] = '/abc:/def' stdout, stderr = run_command(cli.call, '--export-path') expectation = ( 'export PYANG_PLUGINPATH=' + shlex_quote('/abc:/def:{}'.format(dummy_plugin_dir)) ) assert expectation in stdout assert not stderr
def execute_async(self, command, merge_stderr=False, verbose=False): """Start executing command async. Args: command (str): command to execute merge_stderr (bool): merge stderr to stdout verbose (bool): make log records or not Returns: tuple: SSH session, file-like stdin, stdout, stderr """ if verbose: LOGGER.debug("Executing command: '%s'" % command.rstrip()) chan = self._ssh.get_transport().open_session(timeout=self._timeout) chan.set_combine_stderr(merge_stderr) stdin = chan.makefile('wb') stdout = chan.makefile('rb') stderr = chan.makefile_stderr('rb') if self._sudo: command = "sudo -s $SHELL -c " + moves.shlex_quote(command) chan.exec_command(command) return chan, stdin, stdout, stderr
def upload(self, container_name, object_name, check=True): """Step to upload object to container. Args: container_name (str): object storage container object_name (str): name of object to upload check (bool): flag whether to check result or not Raises: AnsibleExecutionException: if command execution failed """ cmd = 'swift upload ' cmd += '{} {}'.format(moves.shlex_quote(container_name), moves.shlex_quote(object_name)) cmd_to_create_object = 'dd if=/dev/zero of={} bs=1M count=1'.format( object_name) self.execute_command(cmd_to_create_object) exit_code, stdout, stderr = self.execute_command(cmd, check=check) if check: assert_that(object_name, equal_to(stdout)) cmd_to_delete_object = 'rm {}'.format(object_name) self.execute_command(cmd_to_delete_object)
def rename_volume(self, volume, name=None, description=None, check=True): """Step to change volume's name or description using CLI. Args: volume (object): cinder volume to edit name (str): new volume name description (str): new volume description check (bool): flag whether to check step or not """ err_msg = 'One of `name` or `description` should be passed.' assert_that(any([name, description]), is_(True), err_msg) cmd = 'cinder rename' if description is not None: cmd += ' --description ' + moves.shlex_quote(description) cmd += ' ' + volume.id if name: cmd += ' ' + name self.execute_command(cmd, timeout=config.VOLUME_AVAILABLE_TIMEOUT, check=check)
def new_cmd(session, name, cmd, mode, logdir, shell): if isinstance(cmd, (list, tuple)): cmd = " ".join(shlex_quote(str(v)) for v in cmd) if name != "htop": cmd += " > {} 2>&1".format(os.path.join(logdir, "logs", "{}.txt".format(name))) if mode == 'tmux': return name, "tmux send-keys -t {}:{} {} Enter".format(session, name, shlex_quote(cmd)) elif mode == 'child': return name, "{} >{}/{}.{}.out 2>&1 & echo kill $! >>{}/kill.sh".format( cmd, logdir, session, name, logdir) elif mode == 'nohup': return name, "nohup {} -c {} >{}/{}.{}.out 2>&1 & echo kill $! >>{}/kill.sh".format( shell, shlex_quote(cmd), logdir, session, name, logdir)
def main(config_file, cloner, args): config.setup_config(config_file) clonemap, repos = get_clonemap_and_repos() with tempfile.NamedTemporaryFile('w+') as f: yaml.dump(clonemap, f, default_flow_style=False) f.flush() f.seek(0) print("Dumped clonemap at", f.name, ":", file=sys.stderr) print(f.read(), file=sys.stderr) cmd = [cloner, '-m', f.name] + args + repos print("Running command:", ' '.join(map(shlex_quote, cmd)), file=sys.stderr) subprocess.check_call(cmd)
def test_reqs_alias_file(d): reqs_file = d.write_to('reqs', [shlex_quote('simple,'+get_exists_path('libsimple'))]) d.cmds(install_cmds(url='--file {}'.format(reqs_file), lib='simple', alias='simple'))
def test_reqs_file(d): reqs_file = d.write_to('reqs', [shlex_quote(get_exists_path('libsimple'))]) d.cmds(install_cmds(url='--file {}'.format(reqs_file), lib='simple', alias=get_exists_path('libsimple')))
def test_reqs_alias_f(d): reqs_file = d.write_to('reqs', [shlex_quote('simple,'+get_exists_path('libsimple'))]) d.cmds(install_cmds(url='-f {}'.format(reqs_file), lib='simple', alias='simple'))
def test_reqs_f(d): reqs_file = d.write_to('reqs', [shlex_quote(get_exists_path('libsimple'))]) d.cmds(install_cmds(url='-f {}'.format(reqs_file), lib='simple', alias=get_exists_path('libsimple')))
def test_reqs_f3(d): reqs_file = d.write_to('reqs', [shlex_quote(get_exists_path('libsimple'))]) d.write_to('requirements.txt', [shlex_quote(get_exists_path('basicapp'))]) d.cmds(install_cmds(url='-f {}'.format(reqs_file), lib='simple', alias=get_exists_path('libsimple')))
def test_without_reqs_f(d): d.write_to('requirements.txt', [shlex_quote(get_exists_path('libsimple'))]) d.cmds(install_cmds(url='', lib='simple', alias=get_exists_path('libsimple')))
def test_without_reqs_f2(d): d.write_to('requirements.txt', [shlex_quote(get_exists_path('basicapp'))]) d.cmds(install_cmds(url=get_exists_path('libsimple'), lib='simple'))
def test_reqs_hash_fail(d): ar = d.get_path('libsimple.tar.gz') create_ar(archive=ar, src=get_exists_path('libsimple')) h = 'xxx' reqs_file = d.write_to('reqs', ["{0} --hash=sha1:{1}".format(shlex_quote(ar), h)]) d.cmds(install_cmds(url='--file {}'.format(reqs_file), lib='simple', alias=ar))
def test_reqs_recipe(d): recipes=shlex_quote(get_exists_path('basicrecipes')) + ' -DCGET_TEST_DIR="' + __test_dir__ + '"' reqs_file = d.write_to('reqs', [recipes, 'simple']) d.cmds(install_cmds(url='--file {}'.format(reqs_file), lib='simple', alias='simple', base_size=1))
def test_flags_reqs_f(d): p = get_exists_path('libsimpleflag') reqs_file = d.write_to('reqs', [shlex_quote(p) + ' -DCGET_FLAG=On']) d.cmds(install_cmds(url='-f {}'.format(reqs_file), alias=p))
def new_cmd(session, name, cmd, mode, logdir, shell): if isinstance(cmd, (list, tuple)): cmd = " ".join(shlex_quote(str(v)) for v in cmd) if mode == 'tmux': return name, "tmux send-keys -t {}:{} {} Enter".format(session, name, shlex_quote(cmd)) elif mode == 'child': return name, "{} >{}/{}.{}.out 2>&1 & echo kill $! >>{}/kill.sh".format(cmd, logdir, session, name, logdir) elif mode == 'nohup': return name, "nohup {} -c {} >{}/{}.{}.out 2>&1 & echo kill $! >>{}/kill.sh".format(shell, shlex_quote(cmd), logdir, session, name, logdir)
def print_out(args): if args.filename: data = yaml.load_file_or_die(args.filename) env_dict = EnvironmentDict.from_yaml_dict(data) else: data = load_user_data_as_yaml_or_die() env_dict = EnvironmentDict.from_yaml_dict(data) if args.only_unencrypted: unencrypted_env_dict = env_dict.remove_all_encrypted(plain=True) else: unencrypted_env_dict = env_dict.decrypt_all_encrypted(plain=True) if args.json: indent = None if args.single_line else 4 print(json.dumps(unencrypted_env_dict, indent=indent, sort_keys=True)) return output = [ '{}={}'.format(key, shlex_quote(value)) for key, value in sorted(unencrypted_env_dict.items()) ] if args.single_line: joiner = ' ' else: joiner = '\n' print(joiner.join(output))
def make_option(self, option, value=None): option = '--' + option if value is not None: option += '=' + shlex_quote(six.text_type(value)) return option
def get_current_values(self, *args, **kwargs): values = super(PlacementPrefOption, self).get_current_values(*args, **kwargs) # noqa for value in values: yield ','.join( u'{strategy}={descriptor}'.format( strategy=strategy.lower(), descriptor=shlex_quote(descriptor[strategy + 'Descriptor']) ) for strategy, descriptor in value.items() )
def save_new_settings(self, settings, image): self.rotate_sentinel_images() labels = [(self.compose_label, settings)] with contextlib.suppress(host_errors): images = self.get_images() images and labels.append((self.images_info_label, images)) dockerfile = ( 'FROM {image}\n' 'LABEL {labels}\n' ).format( image=image or 'scratch', labels=' '.join(itertools.starmap('{0}={1}'.format, labels)), ) build_command = 'echo {dockerfile} | docker build --tag {tag} -'.format( dockerfile=shlex_quote(dockerfile), tag=self.current_settings_tag, ) try: fabricio.run(build_command) except host_errors as error: fabricio.log( 'WARNING: {error}'.format(error=error), output=sys.stderr, color=colors.red, )
def new_cmd(session, name, cmd, monitor, log_dir, shell): if isinstance(cmd, (list, tuple)): cmd = ' '.join(shlex_quote(str(v)) for v in cmd) if monitor == 'tmux': return name, 'tmux send-keys -t {}:{} {} Enter'. \ format(session, name, shlex_quote(cmd)) elif monitor == 'child': return name, '{} >{}/{}.{}.out 2>&1 & echo kill $! >>{}/kill.sh'. \ format(cmd, log_dir, session, name, log_dir) elif monitor == 'nohup': return name, 'nohup {} -c {} >{}/{}.{}.out 2>&1 & echo kill $! >>{}/kill.sh'. \ format(shell, shlex_quote(cmd), log_dir, session, name, log_dir)
def command(self, task_input, task_output, extra_task_args=[]): def quote_and_join(arguments): return " ".join([quote(arg) for arg in arguments]) pieces = [] run_pip = quote(os.path.join(self.path_prefix, 'pip')) run_task = quote( os.path.join(self.path_prefix, '_kubeface-run-task')) kubeface_install_command = self.kubeface_install_command.format( pip=run_pip) if self.kubeface_install_policy == 'if-not-present': # From: http://stackoverflow.com/questions/592620/check-if-a-program-exists-from-a-bash-script pieces.append("command -v %s || { %s ; } " % ( run_task, kubeface_install_command)) elif self.kubeface_install_policy == 'always': pieces.append(kubeface_install_command) if self.pip_packages: pieces.append("%s install %s" % ( run_pip, quote_and_join(self.pip_packages))) pieces.append( run_task + " " + quote_and_join([ task_input, task_output, "--verbose", ] + extra_task_args)) result = " && ".join(pieces) return result
def _cmdlineArg(self): arg = None if self.actionClass in ('_StoreTrueAction', '_StoreFalseAction'): arg = self.option # just the flag itself elif self.option[0] not in ('-', '+'): arg = shlex_quote(self.value) # positional; just the value elif self.value: arg = "%s %s" % (self.option, shlex_quote(self.value)) return arg
def test_print_path(dummy_plugin_dir, register_dummy_plugin, run_command): """ print_path should produce the composed plugin path """ register_dummy_plugin() os.environ['PYANG_PLUGINPATH'] = '/abc:/def' stdout, stderr = run_command(cli.call, '--path') expectation = shlex_quote('/abc:/def:{}'.format(dummy_plugin_dir)) assert expectation in stdout assert not stderr
def create(self, container_name, check=True): """Step to create swift container. Args: container_name (str): name of created container check (bool): flag whether to check result or not Raises: AnsibleExecutionException: if command execution failed """ cmd = 'swift post ' cmd += moves.shlex_quote(container_name) self.execute_command(cmd, check=check)
def list(self, container_name=None, check=True): """Step to get swift list. Args: container_name (str): object storage container check (bool): flag whether to check result or not Raises: AnsibleExecutionException: if command execution failed """ cmd = 'swift list ' if container_name: cmd += moves.shlex_quote(container_name) exit_code, stdout, stderr = self.execute_command(cmd, check=check) return stdout
def create_volume(self, size=1, name=None, description=None, image=None, check=True): """Step to create volume using CLI. Args: size(int): size of created volume (in GB) name (str): name of created volume description (str): volume description image (str): glance image name or ID to create volume from metadata(str): volume metadata check (bool): flag whether to check step or not Returns: dict: cinder volume """ metadata = '{0}={1}'.format(config.STEPLER_PREFIX, config.STEPLER_PREFIX) cmd = 'cinder create ' + str(size) + ' --metadata ' + metadata if image: cmd += ' --image ' + image if name: cmd += ' --name ' + moves.shlex_quote(name) if description is not None: cmd += ' --description ' + moves.shlex_quote(description) exit_code, stdout, stderr = self.execute_command( cmd, timeout=config.VOLUME_AVAILABLE_TIMEOUT, check=check) volume_table = output_parser.table(stdout) volume = {key: value for key, value in volume_table['values']} return volume
def create_volume_backup(self, volume, name=None, description=None, container=None, check=True): """Step to create volume backup using CLI. Args: volume (object): cinder volume name (str): name of backup to create description (str): description container (str): name of the backup service container check (bool): flag whether to check step or not Returns: dict: cinder volume backup """ cmd = 'cinder backup-create' if name: cmd += ' --name ' + name if description is not None: cmd += ' --description ' + moves.shlex_quote(description) if container: cmd += ' --container ' + container cmd += ' ' + volume.id exit_code, stdout, stderr = self.execute_command( cmd, timeout=config.BACKUP_AVAILABLE_TIMEOUT, check=check) backup_table = output_parser.table(stdout) backup = {key: value for key, value in backup_table['values']} return backup
def create_volume_snapshot(self, volume, name=None, description=None, check=True): """Step to create volume snapshot using CLI. Args: volume (object): cinder volume name (str): name of snapshot to create description (str): snapshot description check (bool): flag whether to check step or not Returns: dict: cinder volume snapshot """ cmd = 'cinder snapshot-create' if name: cmd += ' --name ' + name if description is not None: cmd += ' --description ' + moves.shlex_quote(description) cmd += ' ' + volume.id exit_code, stdout, stderr = self.execute_command( cmd, timeout=config.SNAPSHOT_AVAILABLE_TIMEOUT, check=check) snapshot_table = output_parser.table(stdout) snapshot = {key: value for key, value in snapshot_table['values']} return snapshot
def execute_cmd(self, nodes, cmd, timeout=config.ANSIBLE_EXECUTION_MAX_TIMEOUT, check=True): """Execute provided bash command on nodes. Args: nodes (NodeCollection): nodes to execute command on them cmd (str): bash command to execute timeout (int): seconds to wait command executed check (bool): flag whether check step or not Raises: AssertionError|AnsibleExecutionException: if command execution failed in case of check=True Returns: list: AnsibleExecutionRecord(s) """ cmd = (u"timeout {timeout} " u"bash -c {cmd}").format(timeout=timeout, cmd=moves.shlex_quote(cmd)) task = {'shell': cmd.encode('utf-8')} result = nodes.run_task(task, raise_on_error=False) if check: assert_that( result, only_contains(has_properties(status=config.STATUS_OK))) return result
def get_horizon_cinder_backups(self): """Step to retrieve horizon cinder backup enabled or not. Returns: bool: is horizon cinder backup enabled """ pattern = moves.shlex_quote('^\s+["\']enable_backup["\']\s*:\s*True') cmd = "grep -P {} {}".format(pattern, config.HORIZON_CONFIG_PATH) nodes = self.get_nodes(service_names=[config.HORIZON]) result = self.execute_cmd(nodes, cmd, check=False) return all(node_result.status == config.STATUS_OK for node_result in result)
def new_cmd(session, name, cmd, mode, logdir, shell, redirect_file=None): if isinstance(cmd, (list, tuple)): cmd = " ".join(shlex_quote(str(v)) for v in cmd) if mode == 'tmux': if redirect_file: return name, "tmux send-keys -t {}:{} {} Enter".format(session, name, shlex_quote(cmd+" 2>&1 | tee -a {}".format(redirect_file))) else: return name, "tmux send-keys -t {}:{} {} Enter".format(session, name, shlex_quote(cmd)) elif mode == 'child': return name, "{} >{}/{}.{}.out 2>&1 & echo kill $! >>{}/kill.sh".format(cmd, logdir, session, name, logdir) elif mode == 'nohup': return name, "nohup {} -c {} >{}/{}.{}.out 2>&1 & echo kill $! >>{}/kill.sh".format(shell, shlex_quote(cmd), logdir, session, name, logdir)
def create_commands(session, num_workers, remotes, env_id, logdir, shell='bash', mode='tmux', visualise=False): # for launching the TF workers and for launching tensorboard base_cmd = [ 'CUDA_VISIBLE_DEVICES=', sys.executable, 'worker.py', '--log-dir', logdir, '--env-id', env_id, '--num-workers', str(num_workers)] if visualise: base_cmd += ['--visualise'] if remotes is None: remotes = ["1"] * num_workers else: remotes = remotes.split(',') assert len(remotes) == num_workers cmds_map = [new_cmd(session, "ps", base_cmd + ["--job-name", "ps"], mode, logdir, shell)] for i in range(num_workers): cmds_map += [new_cmd(session, "w-%d" % i, base_cmd + ["--job-name", "worker", "--task", str(i), "--remotes", remotes[i]], mode, logdir, shell)] cmds_map += [new_cmd(session, "eval-worker", base_cmd + ["--job-name", "worker", "--task", str(num_workers), "--remotes", remotes[-1], "--eval"], mode,logdir,shell)] cmds_map += [new_cmd(session, "tb", ["tensorboard", "--logdir", logdir, "--port", "12345"], mode, logdir, shell)] if mode == 'tmux': cmds_map += [new_cmd(session, "htop", ["htop"], mode, logdir, shell)] windows = [v[0] for v in cmds_map] notes = [] cmds = [ "mkdir -p {}".format(logdir), "echo {} {} > {}/cmd.sh".format(sys.executable, ' '.join([shlex_quote(arg) for arg in sys.argv if arg != '-n']), logdir), ] if mode == 'nohup' or mode == 'child': cmds += ["echo '#!/bin/sh' >{}/kill.sh".format(logdir)] notes += ["Run `source {}/kill.sh` to kill the job".format(logdir)] if mode == 'tmux': notes += ["Use `tmux attach -t {}` to watch process output".format(session)] notes += ["Use `tmux kill-session -t {}` to kill the job".format(session)] else: notes += ["Use `tail -f {}/*.out` to watch process output".format(logdir)] notes += ["Point your browser to http://localhost:12345 to see Tensorboard"] if mode == 'tmux': cmds += [ "kill $( lsof -i:12345 -t ) > /dev/null 2>&1", # kill any process using tensorboard's port "kill $( lsof -i:12222-{} -t ) > /dev/null 2>&1".format(num_workers+12222), # kill any processes using ps / worker ports "tmux kill-session -t {}".format(session), "tmux new-session -s {} -n {} -d {}".format(session, windows[0], shell) ] for w in windows[1:]: cmds += ["tmux new-window -t {} -n {} {}".format(session, w, shell)] cmds += ["sleep 1"] for window, cmd in cmds_map: cmds += [cmd] return cmds, notes
def create_commands(session, num_workers, remotes, env_id, logdir, shell='bash', mode='tmux', visualise=False): # for launching the TF workers and for launching tensorboard base_cmd = [ 'CUDA_VISIBLE_DEVICES=', sys.executable, 'worker.py', '--log-dir', logdir, '--env-id', env_id, '--num-workers', str(num_workers)] if visualise: base_cmd += ['--visualise'] if remotes is None: remotes = ["1"] * num_workers else: remotes = remotes.split(',') assert len(remotes) == num_workers cmds_map = [new_cmd(session, "ps", base_cmd + ["--job-name", "ps"], mode, logdir, shell)] for i in range(num_workers): cmds_map += [new_cmd(session, "w-%d" % i, base_cmd + ["--job-name", "worker", "--task", str(i), "--remotes", remotes[i]], mode, logdir, shell)] cmds_map += [new_cmd(session, "tb", ["tensorboard", "--logdir", logdir, "--port", "12345"], mode, logdir, shell)] if mode == 'tmux': cmds_map += [new_cmd(session, "htop", ["htop"], mode, logdir, shell)] windows = [v[0] for v in cmds_map] notes = [] cmds = [ "mkdir -p {}".format(logdir), "echo {} {} > {}/cmd.sh".format(sys.executable, ' '.join([shlex_quote(arg) for arg in sys.argv if arg != '-n']), logdir), ] if mode == 'nohup' or mode == 'child': cmds += ["echo '#!/bin/sh' >{}/kill.sh".format(logdir)] notes += ["Run `source {}/kill.sh` to kill the job".format(logdir)] if mode == 'tmux': notes += ["Use `tmux attach -t {}` to watch process output".format(session)] notes += ["Use `tmux kill-session -t {}` to kill the job".format(session)] else: notes += ["Use `tail -f {}/*.out` to watch process output".format(logdir)] notes += ["Point your browser to http://localhost:12345 to see Tensorboard"] if mode == 'tmux': cmds += [ "kill $( lsof -i:12345 -t ) > /dev/null 2>&1", # kill any process using tensorboard's port "kill $( lsof -i:12222-{} -t ) > /dev/null 2>&1".format(num_workers+12222), # kill any processes using ps / worker ports "tmux kill-session -t {}".format(session), "tmux new-session -s {} -n {} -d {}".format(session, windows[0], shell) ] for w in windows[1:]: cmds += ["tmux new-window -t {} -n {} {}".format(session, w, shell)] cmds += ["sleep 1"] for window, cmd in cmds_map: cmds += [cmd] return cmds, notes
def image_create(self, image_file=None, image_name=None, disk_format=None, container_format=None, api_version=config.CURRENT_GLANCE_VERSION, check=True): """Step to create image. Args: image_file (str|None): image file to be uploaded; it should be located on the same node where CLI is running image_name (str|None): name of created image disk_format (str|None): disk format of image container_format (str|None): container format of image api_version (int): API version of Glance (1 or 2) check (bool): flag whether to check result or not Returns: tuple: execution result (image dict, exit_code, stdout, stderr) Raises: AnsibleExecutionException: if command execution failed """ image = None cmd = 'glance image-create' if image_file: cmd += ' --file ' + moves.shlex_quote(image_file) if image_name: cmd += ' --name ' + moves.shlex_quote(image_name) if disk_format: cmd += ' --disk-format ' + disk_format if container_format: cmd += ' --container-format ' + container_format if 'disk-format' not in cmd and 'container-format' not in cmd: cmd += ' <&-' # otherwise stderr: <stdin: is not a tty\nerror: Must provide # --container-format, --disk-format when using stdin> # This problem is only for remote execution (Ansible) exit_code, stdout, stderr = self.execute_command( cmd, environ={'OS_IMAGE_API_VERSION': api_version}, check=check) if check: image_table = output_parser.table(stdout) image = {key: value for key, value in image_table['values']} return image, exit_code, stdout, stderr
def create_router(self, name=None, project=None, username=None, password=None, distributed=None, expected_error=False, check=True): """Step to create router using CLI. Args: name (str): name of created router project (str): name of the project username (str): user name password (str): user password distributed (bool): flag whether to create DVR or not expected_error (bool): flag whether to expect error during router creation or not check (bool): flag whether to check step or not Returns: tuple: (router or None, exit_code, stdout, stderr) """ name = name or next(utils.generate_ids()) router = None cmd = 'neutron router-create ' + moves.shlex_quote(name) if project: cmd += ' --os-project-name ' + project if username and password: cmd += ' --os-username {0} --os-password {1}'.format(username, password) if distributed is not None: cmd += ' --distributed ' + str(distributed) exit_code, stdout, stderr = self.execute_command( cmd, timeout=config.ROUTER_AVAILABLE_TIMEOUT, check=check) if not expected_error: router_table = output_parser.table(stdout) router = {key: value for key, value in router_table['values']} if check: assert_that(router, is_not(empty())) return router, exit_code, stdout, stderr
def check_string_in_file(self, node, file_name, keyword, non_matching=None, start_line_number_file=None, must_present=True, expected_count=None): """Step to check number of keywords in a textual file on a single node. Args: node (NodeCollection): nodes file_name (str): name of textual file keyword (str): string to search non_matching (str|None): string to be absent in result start_line_number_file (str|None): file path with number of first line for searching must_present (bool): flag that keyword must be present or not expected_count (int|None): expected count of lines containing keyword Raises: AssertionError|AnsibleExecutionException: if command execution failed in case of check=True or real count of lines with keyword is not equal to expected one """ # check that file is exists self.execute_cmd(node, 'ls {}'.format(file_name)) if start_line_number_file: start_line_number_cmd = 'cat ' + start_line_number_file else: start_line_number_cmd = 'echo 0' cmd = "tail -n +$({0}) {1} | grep {2}".format( start_line_number_cmd, file_name, moves.shlex_quote(keyword)) if non_matching: cmd += " | grep -v {0}".format(moves.shlex_quote(non_matching)) if expected_count is None: if must_present: matcher = greater_than(0) else: matcher = 0 else: matcher = expected_count result = self.execute_cmd(node, cmd, check=False) for node_result in result: lines = node_result.payload['stdout_lines'] assert_that(lines, has_length(matcher))
def create_commands(session, num_workers, remotes, env_id, logdir, shell='bash', policy='lstm', mode='tmux', visualise=False): # for launching the TF workers and for launching tensorboard base_cmd = [ 'CUDA_VISIBLE_DEVICES=', sys.executable, 'worker.py', '--log-dir', logdir, '--env-id', env_id, '--num-workers', str(num_workers)] if visualise: base_cmd += ['--visualise'] if remotes is None: remotes = ["1"] * num_workers else: remotes = remotes.split(',') assert len(remotes) == num_workers cmds_map = [new_cmd(session, "ps", base_cmd + ["--job-name", "ps"], mode, logdir, shell)] for i in range(num_workers): cmds_map += [new_cmd(session, "w-%d" % i, base_cmd + ["--job-name", "worker", "--task", str(i), "--remotes", remotes[i], "--policy", policy], mode, logdir, shell)] cmds_map += [new_cmd(session, "tb", ["tensorboard", "--logdir", logdir, "--port", "12345"], mode, logdir, shell)] if mode == 'tmux': cmds_map += [new_cmd(session, "htop", ["htop"], mode, logdir, shell)] windows = [v[0] for v in cmds_map] notes = [] cmds = [ "mkdir -p {}".format(logdir), "echo {} {} > {}/cmd.sh".format(sys.executable, ' '.join([shlex_quote(arg) for arg in sys.argv if arg != '-n']), logdir), ] if mode == 'nohup' or mode == 'child': cmds += ["echo '#!/bin/sh' >{}/kill.sh".format(logdir)] notes += ["Run `source {}/kill.sh` to kill the job".format(logdir)] if mode == 'tmux': notes += ["Use `tmux attach -t {}` to watch process output".format(session)] notes += ["Use `tmux kill-session -t {}` to kill the job".format(session)] else: notes += ["Use `tail -f {}/*.out` to watch process output".format(logdir)] notes += ["Point your browser to http://localhost:12345 to see Tensorboard"] if mode == 'tmux': cmds += [ "kill $( lsof -i:12345 -t ) > /dev/null 2>&1", # kill any process using tensorboard's port "kill $( lsof -i:12222-{} -t ) > /dev/null 2>&1".format(num_workers+12222), # kill any processes using ps / worker ports "tmux kill-session -t {}".format(session), "tmux new-session -s {} -n {} -d {}".format(session, windows[0], shell) ] for w in windows[1:]: cmds += ["tmux new-window -t {} -n {} {}".format(session, w, shell)] cmds += ["sleep 1"] for window, cmd in cmds_map: cmds += [cmd] return cmds, notes
def create_commands(session, num_workers, env_id, logdir, shell='bash', mode='tmux', visualise=False): # for launching the TF workers and for launching tensorboard base_cmd = [ 'CUDA_VISIBLE_DEVICES=', sys.executable, 'worker.py', '--log-dir', logdir, '--env-id', env_id, '--num-workers', str(num_workers)] if visualise: base_cmd += ['--visualise'] cmds_map = [new_cmd(session, "ps", base_cmd + ["--job-name", "ps"], mode, logdir, shell)] for i in range(num_workers): cmds_map += [new_cmd(session, "w-%d" % i, base_cmd + ["--job-name", "worker", "--task", str(i)], mode, logdir, shell)] cmds_map += [new_cmd(session, "tb", ["tensorboard", "--logdir", logdir, "--port", "12345"], mode, logdir, shell)] if mode == 'tmux': cmds_map += [new_cmd(session, "htop", ["htop"], mode, logdir, shell)] windows = [v[0] for v in cmds_map] notes = [] cmds = [ "mkdir -p {}".format(logdir), "echo {} {} > {}/cmd.sh".format(sys.executable, ' '.join([shlex_quote(arg) for arg in sys.argv if arg != '-n']), logdir), ] if mode == 'nohup' or mode == 'child': cmds += ["echo '#!/bin/sh' >{}/kill.sh".format(logdir)] notes += ["Run `source {}/kill.sh` to kill the job".format(logdir)] if mode == 'tmux': notes += ["Use `tmux attach -t {}` to watch process output".format(session)] notes += ["Use `tmux kill-session -t {}` to kill the job".format(session)] else: notes += ["Use `tail -f {}/*.out` to watch process output".format(logdir)] notes += ["Point your browser to http://localhost:12345 to see Tensorboard"] if mode == 'tmux': cmds += [ "kill $( lsof -i:12345 -t ) > /dev/null 2>&1", # kill any process using tensorboard's port "kill $( lsof -i:12222-{} -t ) > /dev/null 2>&1".format(num_workers+12222), # kill any processes using ps / worker ports "tmux kill-session -t {}".format(session), "tmux new-session -s {} -n {} -d {}".format(session, windows[0], shell) ] for w in windows[1:]: cmds += ["tmux new-window -t {} -n {} {}".format(session, w, shell)] cmds += ["sleep 1"] for window, cmd in cmds_map: cmds += [cmd] return cmds, notes
