我们从Python开源项目中，提取了以下43个代码示例，用于说明如何使用numpy.ma.getmaskarray()。
def update_alpha(self, p_y_given_x, theta, Xm, tcs): """A rule for non-tree CorEx structure. """ sample = np.random.choice(np.arange(Xm.shape[0]), min(self.max_samples, Xm.shape[0]), replace=False) p_y_given_x = p_y_given_x[:, sample, :] not_missing = np.logical_not(ma.getmaskarray(Xm[sample])) alpha = np.empty((self.n_hidden, self.n_visible)) n_samples, n_visible = Xm.shape memory_size = float(self.max_samples * n_visible * self.n_hidden * self.dim_hidden * 64) / 1000**3 # GB batch_size = np.clip(int(self.ram * n_visible / memory_size), 1, n_visible) for i in range(0, n_visible, batch_size): log_marg_x = self.calculate_marginals_on_samples(theta[i:i+batch_size], Xm[sample, i:i+batch_size]) correct_predictions = np.argmax(p_y_given_x, axis=2)[:, :, np.newaxis] == np.argmax(log_marg_x, axis=3) for ip in range(i, min(i + batch_size, n_visible)): alpha[:, ip] = self.unique_info(correct_predictions[:, not_missing[:, ip], ip - i].T) for j in np.where(np.abs(tcs) < self.tc_min)[0]: # Priming for un-used hidden units amax = np.clip(np.max(alpha[j, :]), 0.01, 0.99) alpha[j, :] = alpha[j, :]**(np.log(0.99)/np.log(amax)) + 0.001 * np.random.random(self.n_visible) self.alpha = alpha[:, :, np.newaxis] # TODO: This is the "correct" update but it is quite noisy. Add smoothing?
def test_set_mask(self): base = self.base.copy() mbase = base.view(mrecarray) # Set the mask to True ....................... mbase.mask = masked assert_equal(ma.getmaskarray(mbase['b']), [1]*5) assert_equal(mbase['a']._mask, mbase['b']._mask) assert_equal(mbase['a']._mask, mbase['c']._mask) assert_equal(mbase._mask.tolist(), np.array([(1, 1, 1)]*5, dtype=bool)) # Delete the mask ............................ mbase.mask = nomask assert_equal(ma.getmaskarray(mbase['c']), [0]*5) assert_equal(mbase._mask.tolist(), np.array([(0, 0, 0)]*5, dtype=bool))
def __setitem__(self, indx, value): """ Sets the given record to value. """ MaskedArray.__setitem__(self, indx, value) if isinstance(indx, basestring): self._mask[indx] = ma.getmaskarray(value)
def fromarrays(arraylist, dtype=None, shape=None, formats=None, names=None, titles=None, aligned=False, byteorder=None, fill_value=None): """ Creates a mrecarray from a (flat) list of masked arrays. Parameters ---------- arraylist : sequence A list of (masked) arrays. Each element of the sequence is first converted to a masked array if needed. If a 2D array is passed as argument, it is processed line by line dtype : {None, dtype}, optional Data type descriptor. shape : {None, integer}, optional Number of records. If None, shape is defined from the shape of the first array in the list. formats : {None, sequence}, optional Sequence of formats for each individual field. If None, the formats will be autodetected by inspecting the fields and selecting the highest dtype possible. names : {None, sequence}, optional Sequence of the names of each field. fill_value : {None, sequence}, optional Sequence of data to be used as filling values. Notes ----- Lists of tuples should be preferred over lists of lists for faster processing. """ datalist = [getdata(x) for x in arraylist] masklist = [np.atleast_1d(getmaskarray(x)) for x in arraylist] _array = recfromarrays(datalist, dtype=dtype, shape=shape, formats=formats, names=names, titles=titles, aligned=aligned, byteorder=byteorder).view(mrecarray) _array._mask.flat = list(zip(*masklist)) if fill_value is not None: _array.fill_value = fill_value return _array
def calculate_theta(self, Xm, p_y_given_x): """Estimate marginal parameters from data and expected latent labels.""" theta = [] for i in range(self.n_visible): not_missing = np.logical_not(ma.getmaskarray(Xm)[:, i]) theta.append(self.estimate_parameters(Xm.data[not_missing, i], p_y_given_x[:, not_missing])) return np.array(theta)
def calculate_p_xi_given_y(self, xi, thetai): not_missing = np.logical_not(ma.getmaskarray(xi)) z = np.zeros((self.n_hidden, len(xi), self.dim_hidden)) z[:, not_missing, :] = self.marginal_p(xi[not_missing], thetai) return z # n_hidden, n_samples, dim_hidden
def calculate_mis(self, p_y_given_x, theta, Xm): mis = np.zeros((self.n_hidden, self.n_visible)) sample = np.random.choice(np.arange(Xm.shape[0]), min(self.max_samples, Xm.shape[0]), replace=False) n_observed = np.sum(np.logical_not(ma.getmaskarray(Xm[sample])), axis=0) n_samples, n_visible = Xm.shape memory_size = float(n_samples * n_visible * self.n_hidden * self.dim_hidden * 64) / 1000**3 # GB batch_size = np.clip(int(self.ram * n_visible / memory_size), 1, n_visible) for i in range(0, n_visible, batch_size): log_marg_x = self.calculate_marginals_on_samples(theta[i:i+batch_size, ...], Xm[sample, i:i+batch_size]) # n_hidden, n_samples, n_visible, dim_hidden mis[:, i:i+batch_size] = np.einsum('ijl,ijkl->ik', p_y_given_x[:, sample, :], log_marg_x) / n_observed[i:i+batch_size][np.newaxis, :] return mis # MI in nats
def _masked_rec_array_to_mgr(data, index, columns, dtype, copy): """ extract from a masked rec array and create the manager """ # essentially process a record array then fill it fill_value = data.fill_value fdata = ma.getdata(data) if index is None: index = _get_names_from_index(fdata) if index is None: index = _default_index(len(data)) index = _ensure_index(index) if columns is not None: columns = _ensure_index(columns) arrays, arr_columns = _to_arrays(fdata, columns) # fill if needed new_arrays = [] for fv, arr, col in zip(fill_value, arrays, arr_columns): mask = ma.getmaskarray(data[col]) if mask.any(): arr, fv = _maybe_upcast(arr, fill_value=fv, copy=True) arr[mask] = fv new_arrays.append(arr) # create the manager arrays, arr_columns = _reorder_arrays(new_arrays, arr_columns, columns) if columns is None: columns = arr_columns mgr = _arrays_to_mgr(arrays, arr_columns, index, columns) if copy: mgr = mgr.copy() return mgr
def addfield(mrecord, newfield, newfieldname=None): """Adds a new field to the masked record array Uses `newfield` as data and `newfieldname` as name. If `newfieldname` is None, the new field name is set to 'fi', where `i` is the number of existing fields. """ _data = mrecord._data _mask = mrecord._mask if newfieldname is None or newfieldname in reserved_fields: newfieldname = 'f%i' % len(_data.dtype) newfield = ma.array(newfield) # Get the new data. # Create a new empty recarray newdtype = np.dtype(_data.dtype.descr + [(newfieldname, newfield.dtype)]) newdata = recarray(_data.shape, newdtype) # Add the exisintg field [newdata.setfield(_data.getfield(*f), *f) for f in _data.dtype.fields.values()] # Add the new field newdata.setfield(newfield._data, *newdata.dtype.fields[newfieldname]) newdata = newdata.view(MaskedRecords) # Get the new mask # Create a new empty recarray newmdtype = np.dtype([(n, bool_) for n in newdtype.names]) newmask = recarray(_data.shape, newmdtype) # Add the old masks [newmask.setfield(_mask.getfield(*f), *f) for f in _mask.dtype.fields.values()] # Add the mask of the new field newmask.setfield(getmaskarray(newfield), *newmask.dtype.fields[newfieldname]) newdata._mask = newmask return newdata
def test_no_data_available(): """ This is a position without valid data """ db = WOA() out = db['sea_water_temperature'].extract( doy=155, lat=48.1953, lon=-69.5855, depth=[2.0, 5.0, 6.0, 21.0, 44.0, 79.0, 5000]) assert sorted(out.keys()) == [u't_dd', u't_mn', u't_sd', u't_se'] for v in out: ma.getmaskarray(out[v]).all()
def test_set_fields(self): # Tests setting fields. base = self.base.copy() mbase = base.view(mrecarray) mbase = mbase.copy() mbase.fill_value = (999999, 1e20, 'N/A') # Change the data, the mask should be conserved mbase.a._data[:] = 5 assert_equal(mbase['a']._data, [5, 5, 5, 5, 5]) assert_equal(mbase['a']._mask, [0, 1, 0, 0, 1]) # Change the elements, and the mask will follow mbase.a = 1 assert_equal(mbase['a']._data, [1]*5) assert_equal(ma.getmaskarray(mbase['a']), [0]*5) # Use to be _mask, now it's recordmask assert_equal(mbase.recordmask, [False]*5) assert_equal(mbase._mask.tolist(), np.array([(0, 0, 0), (0, 1, 1), (0, 0, 0), (0, 0, 0), (0, 1, 1)], dtype=bool)) # Set a field to mask ........................ mbase.c = masked # Use to be mask, and now it's still mask ! assert_equal(mbase.c.mask, [1]*5) assert_equal(mbase.c.recordmask, [1]*5) assert_equal(ma.getmaskarray(mbase['c']), [1]*5) assert_equal(ma.getdata(mbase['c']), [asbytes('N/A')]*5) assert_equal(mbase._mask.tolist(), np.array([(0, 0, 1), (0, 1, 1), (0, 0, 1), (0, 0, 1), (0, 1, 1)], dtype=bool)) # Set fields by slices ....................... mbase = base.view(mrecarray).copy() mbase.a[3:] = 5 assert_equal(mbase.a, [1, 2, 3, 5, 5]) assert_equal(mbase.a._mask, [0, 1, 0, 0, 0]) mbase.b[3:] = masked assert_equal(mbase.b, base['b']) assert_equal(mbase.b._mask, [0, 1, 0, 1, 1]) # Set fields globally.......................... ndtype = [('alpha', '|S1'), ('num', int)] data = ma.array([('a', 1), ('b', 2), ('c', 3)], dtype=ndtype) rdata = data.view(MaskedRecords) val = ma.array([10, 20, 30], mask=[1, 0, 0]) with warnings.catch_warnings(): warnings.simplefilter("ignore") rdata['num'] = val assert_equal(rdata.num, val) assert_equal(rdata.num.mask, [1, 0, 0])
def __setattr__(self, attr, val): """ Sets the attribute attr to the value val. """ # Should we call __setmask__ first ? if attr in ['mask', 'fieldmask']: self.__setmask__(val) return # Create a shortcut (so that we don't have to call getattr all the time) _localdict = object.__getattribute__(self, '__dict__') # Check whether we're creating a new field newattr = attr not in _localdict try: # Is attr a generic attribute ? ret = object.__setattr__(self, attr, val) except: # Not a generic attribute: exit if it's not a valid field fielddict = ndarray.__getattribute__(self, 'dtype').fields or {} optinfo = ndarray.__getattribute__(self, '_optinfo') or {} if not (attr in fielddict or attr in optinfo): exctype, value = sys.exc_info()[:2] raise exctype(value) else: # Get the list of names fielddict = ndarray.__getattribute__(self, 'dtype').fields or {} # Check the attribute if attr not in fielddict: return ret if newattr: # We just added this one or this setattr worked on an # internal attribute. try: object.__delattr__(self, attr) except: return ret # Let's try to set the field try: res = fielddict[attr][:2] except (TypeError, KeyError): raise AttributeError("record array has no attribute %s" % attr) if val is masked: _fill_value = _localdict['_fill_value'] if _fill_value is not None: dval = _localdict['_fill_value'][attr] else: dval = val mval = True else: dval = filled(val) mval = getmaskarray(val) obj = ndarray.__getattribute__(self, '_data').setfield(dval, *res) _localdict['_mask'].__setitem__(attr, mval) return obj
def woa_profile_from_dap(var, d, lat, lon, depth, cfg): """ Monthly Climatologic Mean and Standard Deviation from WOA, used either for temperature or salinity. INPUTS time: [day of the year] lat: [-90<lat<90] lon: [-180<lon<180] depth: [meters] Reads the WOA Monthly Climatology NetCDF file and returns the corresponding WOA values of salinity or temperature mean and standard deviation for the given time, lat, lon, depth. """ if lon < 0: lon = lon+360 url = cfg['url'] doy = int(d.strftime('%j')) dataset = open_url(url) dn = (np.abs(doy-dataset['time'][:])).argmin() xn = (np.abs(lon-dataset['lon'][:])).argmin() yn = (np.abs(lat-dataset['lat'][:])).argmin() if re.match("temperature\d?$", var): mn = ma.masked_values(dataset.t_mn.t_mn[dn, :, yn, xn].reshape( dataset['depth'].shape[0]), dataset.t_mn.attributes['_FillValue']) sd = ma.masked_values(dataset.t_sd.t_sd[dn, :, yn, xn].reshape( dataset['depth'].shape[0]), dataset.t_sd.attributes['_FillValue']) # se = ma.masked_values(dataset.t_se.t_se[dn, :, yn, xn].reshape( # dataset['depth'].shape[0]), dataset.t_se.attributes['_FillValue']) # Use this in the future. A minimum # of samples # dd = ma.masked_values(dataset.t_dd.t_dd[dn, :, yn, xn].reshape( # dataset['depth'].shape[0]), dataset.t_dd.attributes['_FillValue']) elif re.match("salinity\d?$", var): mn = ma.masked_values(dataset.s_mn.s_mn[dn, :, yn, xn].reshape( dataset['depth'].shape[0]), dataset.s_mn.attributes['_FillValue']) sd = ma.masked_values(dataset.s_sd.s_sd[dn, :, yn, xn].reshape( dataset['depth'].shape[0]), dataset.s_sd.attributes['_FillValue']) # dd = ma.masked_values(dataset.s_dd.s_dd[dn, :, yn, xn].reshape( # dataset['depth'].shape[0]), dataset.s_dd.attributes['_FillValue']) zwoa = ma.array(dataset.depth[:]) ind = (depth <= zwoa.max()) & (depth >= zwoa.min()) # Mean value profile f = interp1d(zwoa[~ma.getmaskarray(mn)].compressed(), mn.compressed()) mn_interp = ma.masked_all(depth.shape) mn_interp[ind] = f(depth[ind]) # The stdev profile f = interp1d(zwoa[~ma.getmaskarray(sd)].compressed(), sd.compressed()) sd_interp = ma.masked_all(depth.shape) sd_interp[ind] = f(depth[ind]) output = {'woa_an': mn_interp, 'woa_sd': sd_interp} return output
def interpolate(self, lat, lon, var): """ Interpolate each var on the coordinates requested """ subset, dims = self.crop(lat, lon, var) if np.all([y in dims['lat'] for y in lat]) & \ np.all([x in dims['lon'] for x in lon]): yn = np.nonzero([y in lat for y in dims['lat']])[0] xn = np.nonzero([x in lon for x in dims['lon']])[0] output = {} for v in subset: # output[v] = subset[v][dn, zn, yn, xn] # Seriously that this is the way to do it?!!?? output[v] = subset[v][:, xn][yn] return output # The output coordinates shall be created only once. points_out = [] for latn in lat: for lonn in lon: points_out.append([latn, lonn]) points_out = np.array(points_out) output = {} for v in var: output[v] = ma.masked_all( (lat.size, lon.size), dtype=subset[v].dtype) # The valid data idx = np.nonzero(~ma.getmaskarray(subset[v])) if idx[0].size > 0: points = np.array([ dims['lat'][idx[0]], dims['lon'][idx[1]]]).T values = subset[v][idx] # Interpolate along the dimensions that have more than one # position, otherwise it means that the output is exactly # on that coordinate. ind = np.array( [np.unique(points[:, i]).size > 1 for i in range(points.shape[1])]) assert ind.any() values_out = griddata( np.atleast_1d(np.squeeze(points[:, ind])), values, np.atleast_1d(np.squeeze(points_out[:, ind])) ) # Remap the interpolated value back into a 4D array idx = np.isfinite(values_out) for [y, x], out in zip(points_out[idx], values_out[idx]): output[v][y==lat, x==lon] = out return output
def test_set_fields(self): # Tests setting fields. base = self.base.copy() mbase = base.view(mrecarray) mbase = mbase.copy() mbase.fill_value = (999999, 1e20, 'N/A') # Change the data, the mask should be conserved mbase.a._data[:] = 5 assert_equal(mbase['a']._data, [5, 5, 5, 5, 5]) assert_equal(mbase['a']._mask, [0, 1, 0, 0, 1]) # Change the elements, and the mask will follow mbase.a = 1 assert_equal(mbase['a']._data, [1]*5) assert_equal(ma.getmaskarray(mbase['a']), [0]*5) # Use to be _mask, now it's recordmask assert_equal(mbase.recordmask, [False]*5) assert_equal(mbase._mask.tolist(), np.array([(0, 0, 0), (0, 1, 1), (0, 0, 0), (0, 0, 0), (0, 1, 1)], dtype=bool)) # Set a field to mask ........................ mbase.c = masked # Use to be mask, and now it's still mask ! assert_equal(mbase.c.mask, [1]*5) assert_equal(mbase.c.recordmask, [1]*5) assert_equal(ma.getmaskarray(mbase['c']), [1]*5) assert_equal(ma.getdata(mbase['c']), [asbytes('N/A')]*5) assert_equal(mbase._mask.tolist(), np.array([(0, 0, 1), (0, 1, 1), (0, 0, 1), (0, 0, 1), (0, 1, 1)], dtype=bool)) # Set fields by slices ....................... mbase = base.view(mrecarray).copy() mbase.a[3:] = 5 assert_equal(mbase.a, [1, 2, 3, 5, 5]) assert_equal(mbase.a._mask, [0, 1, 0, 0, 0]) mbase.b[3:] = masked assert_equal(mbase.b, base['b']) assert_equal(mbase.b._mask, [0, 1, 0, 1, 1]) # Set fields globally.......................... ndtype = [('alpha', '|S1'), ('num', int)] data = ma.array([('a', 1), ('b', 2), ('c', 3)], dtype=ndtype) rdata = data.view(MaskedRecords) val = ma.array([10, 20, 30], mask=[1, 0, 0]) rdata['num'] = val assert_equal(rdata.num, val) assert_equal(rdata.num.mask, [1, 0, 0])
