我们从Python开源项目中，提取了以下18个代码示例，用于说明如何使用multiprocessing.Semaphore()。
def __init__(self, n_walkers, n_workers=None, gpu_indices=None): if gpu_indices is not None: self.gpu_indices = gpu_indices self.n_workers = len(gpu_indices) else: assert n_workers, "If gpu_indices are not given the n_workers must be given" self.n_workers = n_workers self.gpu_indices = range(n_workers) # make a Queue for free workers, when one is being used it is # popped off and locked self.free_workers = mulproc.Queue() # the semaphore provides the locks on the workers self.lock = mulproc.Semaphore(self.n_workers) # initialize a list to put results in self.results_list = mulproc.Manager().list() for i in range(n_walkers): self.results_list.append(None) # add the free worker indices (not device/gpu indices) to the # free workers queue for i in range(self.n_workers): self.free_workers.put(i)
def test_kill_process_tree(self): """Spin up a process that can't be killed by SIGTERM and make sure it gets killed anyway.""" child_process_killed = multiprocessing.Value('i', 0) process_done = multiprocessing.Semaphore(0) child_pid = multiprocessing.Value('i', 0) setup_done = multiprocessing.Semaphore(0) args = [child_process_killed, child_pid, process_done, setup_done] child = multiprocessing.Process(target=TestHelpers._parent_of_ignores_sigterm, args=args) try: child.start() self.assertTrue(process_done.acquire(timeout=5.0)) self.assertEqual(1, child_process_killed.value) finally: try: os.kill(child_pid.value, signal.SIGKILL) # terminate doesnt work here except OSError: pass
def is_multiprocessing_available(): if (pycam.Utils.get_platform() == pycam.Utils.OSPlatform.WINDOWS) and \ hasattr(sys, "frozen") and sys.frozen: return False try: import multiprocessing # try to initialize a semaphore - this can trigger shm access failures # (e.g. on Debian Lenny with Python 2.6.6) multiprocessing.Semaphore() return True except ImportError: if "missing_module" not in __issued_warnings: log.info("Python's multiprocessing module is missing: disabling parallel processing") __issued_warnings.append("missing_module") except OSError: if "shm_access_failed" not in __issued_warnings: log.info("Python's multiprocessing module failed to acquire read/write access to " "shared memory (shm) - disabling parallel processing") __issued_warnings.append("shm_access_failed") return False
def test_timeout(self): if self.TYPE != 'processes': self.skipTest('test not appropriate for {}'.format(self.TYPE)) sem = self.Semaphore(0) acquire = TimingWrapper(sem.acquire) self.assertEqual(acquire(False), False) self.assertTimingAlmostEqual(acquire.elapsed, 0.0) self.assertEqual(acquire(False, None), False) self.assertTimingAlmostEqual(acquire.elapsed, 0.0) self.assertEqual(acquire(False, TIMEOUT1), False) self.assertTimingAlmostEqual(acquire.elapsed, 0) self.assertEqual(acquire(True, TIMEOUT2), False) self.assertTimingAlmostEqual(acquire.elapsed, TIMEOUT2) self.assertEqual(acquire(timeout=TIMEOUT3), False) self.assertTimingAlmostEqual(acquire.elapsed, TIMEOUT3)
def test_waitfor_timeout(self): # based on test in test/lock_tests.py cond = self.Condition() state = self.Value('i', 0) success = self.Value('i', False) sem = self.Semaphore(0) p = self.Process(target=self._test_waitfor_timeout_f, args=(cond, state, success, sem)) p.daemon = True p.start() self.assertTrue(sem.acquire(timeout=10)) # Only increment 3 times, so state == 4 is never reached. for i in range(3): time.sleep(0.01) with cond: state.value += 1 cond.notify() p.join(5) self.assertTrue(success.value)
def __init__(self, num_processes=1, tasks_per_requests=1, bitmap_size=(64 << 10)): self.to_update_queue = multiprocessing.Queue() self.to_master_queue = multiprocessing.Queue() self.to_master_from_mapserver_queue = multiprocessing.Queue() self.to_master_from_slave_queue = multiprocessing.Queue() self.to_mapserver_queue = multiprocessing.Queue() self.to_slave_queues = [] for i in range(num_processes): self.to_slave_queues.append(multiprocessing.Queue()) self.slave_locks_A = [] self.slave_locks_B = [] for i in range(num_processes): self.slave_locks_A.append(multiprocessing.Lock()) self.slave_locks_B.append(multiprocessing.Lock()) self.slave_locks_B[i].acquire() self.reload_semaphore = multiprocessing.Semaphore(multiprocessing.cpu_count()/2) self.num_processes = num_processes self.tasks_per_requests = tasks_per_requests self.stage_abortion_notifier = multiprocessing.Value('b', False) self.slave_termination = multiprocessing.Value('b', False, lock=False) self.sampling_failed_notifier = multiprocessing.Value('b', False) self.effector_mode = multiprocessing.Value('b', False) self.files = ["/dev/shm/kafl_fuzzer_master_", "/dev/shm/kafl_fuzzer_mapserver_", "/dev/shm/kafl_fuzzer_bitmap_"] self.sizes = [(65 << 10), (65 << 10), bitmap_size] self.tmp_shm = [{}, {}, {}]
def __init__(self, opt, world): super().__init__(opt) self.inner_world = world self.numthreads = opt['numthreads'] self.sync = { # syncronization primitives # semaphores for counting queued examples 'queued_sem': Semaphore(0), # counts num exs to be processed 'threads_sem': Semaphore(0), # counts threads 'reset_sem': Semaphore(0), # allows threads to reset # flags for communicating with threads 'reset_flag': Value('b', False), # threads should reset 'term_flag': Value('b', False), # threads should terminate # counters 'epoch_done_ctr': Value('i', 0), # number of done threads 'total_parleys': Value('l', 0), # number of parleys in threads } # don't let threads create more threads! self.threads = [] for i in range(self.numthreads): self.threads.append(HogwildProcess(i, opt, world, self.sync)) for t in self.threads: t.start() for _ in self.threads: self.sync['threads_sem'].acquire()
def clopure_pmap(self, *args, local_vars): if len(args) <= 1: raise ClopureRuntimeError("pmap takes at least 2 arguments") seqs = [self.evaluate(arg, local_vars=local_vars) for arg in args[1:]] p = Pool(self.procs) s = Semaphore(self.queue_size) input_iter = (((args[0],) + x, local_vars) for x in input_semaphore_hook(zip(*seqs), s)) return output_semaphore_hook(p.imap(self.mp_evaluate_wrapper, input_iter), s)
def clopure_pmap_unord(self, *args, local_vars): if len(args) <= 1: raise ClopureRuntimeError("pmap-unord takes at least 2 arguments") seqs = [self.evaluate(arg, local_vars=local_vars) for arg in args[1:]] p = Pool(self.procs) s = Semaphore(self.queue_size) input_iter = (((args[0],) + x, local_vars) for x in input_semaphore_hook(zip(*seqs), s)) return output_semaphore_hook(p.imap_unordered(self.mp_evaluate_wrapper, input_iter), s)
def clopure_iter_mp_split_unord(self, fn, local_vars): def iter_split_generator(*g): q_in = Queue() q_out = Queue() exit_input_thread = False semaphore = Semaphore(self.queue_size) ps = [Process(target=self.iter_split_evaluate_wrapper, args=(fn, local_vars, len(g), q_in, q_out)) for i in range(self.procs)] for p in ps: p.start() def input_thread(): try: for i, item in enumerate(zip(*g)): semaphore.acquire() if exit_input_thread: return q_in.put((i, item)) except BaseException: traceback.print_exc(file=sys.stdout) for i in range(self.procs): q_in.put((0, EOFMessage)) t = Thread(target=input_thread) t.start() n_working_procs = self.procs while True: k, data = q_out.get() if data is EOFMessage: n_working_procs -= 1 if n_working_procs == 0: break continue yield data semaphore.release() for p in ps: p.join() exit_input_thread = True semaphore.release() return iter_split_generator
def test_kill_using_shell(self): """Test when no process exists.""" child_pid = multiprocessing.Value('i', 0) setup_done = multiprocessing.Semaphore(0) args = [child_pid, setup_done] child = multiprocessing.Process(target=TestHelpers._ignores_sigterm, args=args) child.start() self.assertTrue(setup_done.acquire(timeout=1.0)) pid_to_kill = child_pid.value self.assertTrue(helpers.kill_using_shell(logging.getLogger(), pid_to_kill, signal=signal.SIGKILL)) child.join() # remove orphan process self.assertFalse(helpers.kill_using_shell(logging.getLogger(), pid_to_kill, signal=signal.SIGKILL))
def __init__(self, n): self.n = n self.counter = SharedCounter(0) self.barrier = Semaphore(0)
def __init__(self, forks_number): self.forks_number = forks_number self.semaphore = Semaphore(self.forks_number)
def test_semaphore(self): sem = self.Semaphore(2) self._test_semaphore(sem) self.assertEqual(sem.release(), None) self.assertReturnsIfImplemented(3, get_value, sem) self.assertEqual(sem.release(), None) self.assertReturnsIfImplemented(4, get_value, sem)
def clopure_iter_mp_split(self, fn, local_vars): def iter_split_generator(*g): q_in = Queue() q_out = Queue() exit_input_thread = False semaphore = Semaphore(self.queue_size) ps = [Process(target=self.iter_split_evaluate_wrapper, args=(fn, local_vars, len(g), q_in, q_out)) for i in range(self.procs)] for p in ps: p.start() def input_thread(): try: for i, item in enumerate(zip(*g)): semaphore.acquire() if exit_input_thread: return q_in.put((i, item)) except BaseException: traceback.print_exc(file=sys.stdout) for i in range(self.procs): q_in.put((0, EOFMessage)) t = Thread(target=input_thread) t.start() cur = 0 n_working_procs = self.procs l = [None] * self.queue_size while True: k, data = q_out.get() if data is EOFMessage: n_working_procs -= 1 if n_working_procs == 0: break continue l[k - cur] = (k, data) while l[0]: yield l.pop(0)[1] l.append(None) cur += 1 semaphore.release() exit_input_thread = True semaphore.release() return iter_split_generator
def test_notify(self): cond = self.Condition() sleeping = self.Semaphore(0) woken = self.Semaphore(0) p = self.Process(target=self.f, args=(cond, sleeping, woken)) p.daemon = True p.start() p = threading.Thread(target=self.f, args=(cond, sleeping, woken)) p.daemon = True p.start() # wait for both children to start sleeping sleeping.acquire() sleeping.acquire() # check no process/thread has woken up time.sleep(DELTA) self.assertReturnsIfImplemented(0, get_value, woken) # wake up one process/thread cond.acquire() cond.notify() cond.release() # check one process/thread has woken up time.sleep(DELTA) self.assertReturnsIfImplemented(1, get_value, woken) # wake up another cond.acquire() cond.notify() cond.release() # check other has woken up time.sleep(DELTA) self.assertReturnsIfImplemented(2, get_value, woken) # check state is not mucked up self.check_invariant(cond) p.join()
def test_wait_integer(self): from multiprocessing.connection import wait expected = 3 sorted_ = lambda l: sorted(l, key=lambda x: id(x)) sem = multiprocessing.Semaphore(0) a, b = multiprocessing.Pipe() p = multiprocessing.Process(target=self.signal_and_sleep, args=(sem, expected)) p.start() self.assertIsInstance(p.sentinel, int) self.assertTrue(sem.acquire(timeout=20)) start = time.time() res = wait([a, p.sentinel, b], expected + 20) delta = time.time() - start self.assertEqual(res, [p.sentinel]) self.assertLess(delta, expected + 2) self.assertGreater(delta, expected - 2) a.send(None) start = time.time() res = wait([a, p.sentinel, b], 20) delta = time.time() - start self.assertEqual(sorted_(res), sorted_([p.sentinel, b])) self.assertLess(delta, 0.4) b.send(None) start = time.time() res = wait([a, p.sentinel, b], 20) delta = time.time() - start self.assertEqual(sorted_(res), sorted_([a, p.sentinel, b])) self.assertLess(delta, 0.4) p.terminate() p.join()
