/** * Get task clocks for all matrices from Master * @return task clocks for all matrices from Master * @throws ServiceException */ public Int2ObjectOpenHashMap<Int2IntOpenHashMap> getTaskMatrixClocks() throws ServiceException { GetTaskMatrixClockResponse response = masterProxy.getTaskMatrixClocks(null, GetTaskMatrixClockRequest.newBuilder().build()); Int2ObjectOpenHashMap<Int2IntOpenHashMap> taskIdToMatrixClocksMap = new Int2ObjectOpenHashMap<>(response.getTaskMatrixClocksCount()); List<TaskMatrixClock> taskMatrixClocks = response.getTaskMatrixClocksList(); int size = taskMatrixClocks.size(); int matrixNum; for(int i = 0; i < size; i++) { Int2IntOpenHashMap matrixIdToClockMap = new Int2IntOpenHashMap(taskMatrixClocks.get(i).getMatrixClocksCount()); taskIdToMatrixClocksMap.put(taskMatrixClocks.get(i).getTaskId().getTaskIndex(), matrixIdToClockMap); List<MatrixClock> matrixClocks = taskMatrixClocks.get(i).getMatrixClocksList(); matrixNum = matrixClocks.size(); for(int j = 0; j < matrixNum; j++) { matrixIdToClockMap.put(matrixClocks.get(j).getMatrixId(), matrixClocks.get(j).getClock()); } } return taskIdToMatrixClocksMap; }
/** * Adjust clock values * @param taskToMatrixClocks taskId->(matrixId->clock) map */ public void adjustClocks(Int2ObjectOpenHashMap<Int2IntOpenHashMap> taskToMatrixClocks) { ObjectIterator<Int2ObjectMap.Entry<Int2IntOpenHashMap>> taskIter = taskToMatrixClocks.int2ObjectEntrySet().fastIterator(); Int2ObjectMap.Entry<Int2IntOpenHashMap> taskEntry = null; int taskId = 0; Int2IntOpenHashMap matrixIdToClockMap = null; ObjectIterator<Int2IntMap.Entry> matrixIter = null; Int2IntMap.Entry matrixEntry = null; while(taskIter.hasNext()) { taskEntry = taskIter.next(); taskId = taskEntry.getIntKey(); matrixIdToClockMap = taskEntry.getValue(); matrixIter = matrixIdToClockMap.int2IntEntrySet().fastIterator(); while (matrixIter.hasNext()) { matrixEntry = matrixIter.next(); updateClock(matrixEntry.getIntKey(), taskId, matrixEntry.getIntValue()); } } }
protected void initializeAttributes() throws AlgorithmExecutionException { this.numColumns = this.columnNames.size(); this.attributeId2attributeObject = new Int2ObjectOpenHashMap<Attribute>(this.numColumns); this.attributeObjectQueue = new PriorityQueue<Attribute>(this.numColumns); for (int table = 0; table < this.tableNames.length; table++) { int firstAttribute = this.tableColumnStartIndexes[table]; int lastAttribute = (table == this.tableNames.length - 1) ? this.numColumns : this.tableColumnStartIndexes[table + 1]; for (int attribute = firstAttribute; attribute < lastAttribute; attribute++) { Attribute spiderAttribute; if (this.databaseConnectionGenerator != null) spiderAttribute = new Attribute(attribute, this.columnTypes, this.databaseConnectionGenerator, this.inputRowLimit, this.dao, this.tableNames[table], this.columnNames.get(attribute), this.tempFolder); else spiderAttribute = new Attribute(attribute, this.columnTypes, this.fileInputGenerator[table], this.inputRowLimit, attribute - firstAttribute, this.tempFolder, this.maxMemoryUsage, this.memoryCheckFrequency); this.attributeId2attributeObject.put(attribute, spiderAttribute); if (!spiderAttribute.hasFinished()) this.attributeObjectQueue.add(spiderAttribute); } } }
private void output() throws CouldNotReceiveResultException, ColumnNameMismatchException { // Read the discovered INDs from the attributes Int2ObjectOpenHashMap<IntList> dep2ref = new Int2ObjectOpenHashMap<IntList>(this.numColumns); for (Attribute spiderAttribute : this.attributeId2attributeObject.values()) if (!spiderAttribute.getReferenced().isEmpty()) dep2ref.put(spiderAttribute.getAttributeId(), new IntArrayList(spiderAttribute.getReferenced())); // Write the result to the resultReceiver for (int dep : dep2ref.keySet()) { String depTableName = this.getTableNameFor(dep, this.tableColumnStartIndexes); String depColumnName = this.columnNames.get(dep); for (int ref : dep2ref.get(dep)) { String refTableName = this.getTableNameFor(ref, this.tableColumnStartIndexes); String refColumnName = this.columnNames.get(ref); this.resultReceiver.receiveResult(new InclusionDependency(new ColumnPermutation(new ColumnIdentifier(depTableName, depColumnName)), new ColumnPermutation(new ColumnIdentifier(refTableName, refColumnName)))); this.numUnaryINDs++; } } }
public void generalize() { int maxLevel = this.numAttributes; // Build an index level->nodes for the top-down, level-wise traversal Int2ObjectOpenHashMap<ArrayList<ElementLhsPair>> level2elements = new Int2ObjectOpenHashMap<>(maxLevel); for (int level = 0; level < maxLevel; level++) level2elements.put(level, new ArrayList<ElementLhsPair>()); this.addToIndex(level2elements, 0, new OpenBitSet(this.numAttributes)); // Traverse the levels top-down and add all direct generalizations for (int level = maxLevel - 1; level >= 0; level--) { for (ElementLhsPair pair : level2elements.get(level)) { // Remove isFDs, because we will mark valid FDs later on pair.element.removeAllFds(); // Generate and add generalizations for (int lhsAttr = pair.lhs.nextSetBit(0); lhsAttr >= 0; lhsAttr = pair.lhs.nextSetBit(lhsAttr + 1)) { pair.lhs.clear(lhsAttr); FDTreeElement generalization = this.addGeneralization(pair.lhs, pair.element.getRhsAttributes()); if (generalization != null) level2elements.get(level - 1).add(new ElementLhsPair(generalization, pair.lhs.clone())); pair.lhs.set(lhsAttr); } } } }
protected static Entry<Map<String, SimpleEnum<?>>, Int2ObjectMap<SimpleEnum<?>>> init(final Class<?> clazz, final int size) { Map<String, SimpleEnum<?>> byName = ASimpleEnum.byName.get(clazz); if (byName == null) { byName = new CaseInsensitiveMap<>(size, SMALL_LOAD_FACTOR); ASimpleEnum.byName.put(clazz, byName); } Int2ObjectMap<SimpleEnum<?>> byID = ASimpleEnum.byOrdinal.get(clazz); if (byID == null) { byID = new Int2ObjectOpenHashMap<>(size, SMALL_LOAD_FACTOR); ASimpleEnum.byOrdinal.put(clazz, byID); } return new SimpleEntry<>(byName, byID); }
private void replayTrain(@Nonnull final ByteBuffer buf) { final int itemI = buf.getInt(); final int knnSize = buf.getInt(); final Int2ObjectMap<Int2FloatMap> knnItems = new Int2ObjectOpenHashMap<>(1024); final IntSet pairItems = new IntOpenHashSet(); for (int i = 0; i < knnSize; i++) { int user = buf.getInt(); int ruSize = buf.getInt(); Int2FloatMap ru = new Int2FloatOpenHashMap(ruSize); ru.defaultReturnValue(0.f); for (int j = 0; j < ruSize; j++) { int itemK = buf.getInt(); pairItems.add(itemK); float ruk = buf.getFloat(); ru.put(itemK, ruk); } knnItems.put(user, ru); } for (int itemJ : pairItems) { train(itemI, knnItems, itemJ); } }
@Nonnull private static Int2ObjectMap<Int2FloatMap> kNNentries(@Nonnull final Object kNNiObj, @Nonnull final MapObjectInspector knnItemsOI, @Nonnull final PrimitiveObjectInspector knnItemsKeyOI, @Nonnull final MapObjectInspector knnItemsValueOI, @Nonnull final PrimitiveObjectInspector knnItemsValueKeyOI, @Nonnull final PrimitiveObjectInspector knnItemsValueValueOI, @Nullable Int2ObjectMap<Int2FloatMap> knnItems, @Nonnull final MutableInt nnzKNNi) { if (knnItems == null) { knnItems = new Int2ObjectOpenHashMap<>(1024); } else { knnItems.clear(); } int numElementOfKNNItems = 0; for (Map.Entry<?, ?> entry : knnItemsOI.getMap(kNNiObj).entrySet()) { int user = PrimitiveObjectInspectorUtils.getInt(entry.getKey(), knnItemsKeyOI); Int2FloatMap ru = int2floatMap(knnItemsValueOI.getMap(entry.getValue()), knnItemsValueKeyOI, knnItemsValueValueOI); knnItems.put(user, ru); numElementOfKNNItems += ru.size(); } nnzKNNi.setValue(numElementOfKNNItems); return knnItems; }
protected void calculateLayerArrays() { Int2ObjectOpenHashMap<char[]> colorLayerMap = new Int2ObjectOpenHashMap<>(); for (int i = 0; i < validBlockIds.length; i++) { int color = validColors[i]; int combined = validBlockIds[i]; if (hasAlpha(color)) { for (int j = 0; j < validBlockIds.length; j++) { int colorOther = validColors[j]; if (!hasAlpha(colorOther)) { int combinedOther = validBlockIds[j]; int combinedColor = combineTransparency(color, colorOther); colorLayerMap.put(combinedColor, new char[]{(char) combined, (char) combinedOther}); } } } } this.validLayerColors = new int[colorLayerMap.size()]; this.validLayerBlocks = new char[colorLayerMap.size()][]; int index = 0; for (Int2ObjectMap.Entry<char[]> entry : colorLayerMap.int2ObjectEntrySet()) { validLayerColors[index] = entry.getIntKey(); validLayerBlocks[index++] = entry.getValue(); } }
/** * Get the candidates: points which have at least one hash bucket in common. * * @param obj Query object * @return Candidates */ protected DBIDs getCandidates(V obj) { ModifiableDBIDs candidates = null; final int numhash = hashtables.size(); double[] buf = new double[hashfunctions.get(0).getNumberOfProjections()]; for(int i = 0; i < numhash; i++) { final Int2ObjectOpenHashMap<DBIDs> table = hashtables.get(i); final LocalitySensitiveHashFunction<? super V> hashfunc = hashfunctions.get(i); // Get the initial (unbounded) hash code: int hash = hashfunc.hashObject(obj, buf); // Reduce to hash table size int bucket = hash % numberOfBuckets; DBIDs cur = table.get(bucket); if(cur != null) { if(candidates == null) { candidates = DBIDUtil.newHashSet(cur.size() * numhash); } candidates.addDBIDs(cur); } } return (candidates == null) ? DBIDUtil.EMPTYDBIDS : candidates; }
/** * Run the algorithm on a database. * * @param db Database * @param relation Relation to process. * @return Hierarchical result */ public PointerPrototypeHierarchyRepresentationResult run(Database db, Relation<O> relation) { DistanceQuery<O> dq = DatabaseUtil.precomputedDistanceQuery(db, relation, getDistanceFunction(), LOG); final DBIDs ids = relation.getDBIDs(); final int size = ids.size(); // Initialize space for result: PointerHierarchyRepresentationBuilder builder = new PointerHierarchyRepresentationBuilder(ids, dq.getDistanceFunction().isSquared()); Int2ObjectOpenHashMap<ModifiableDBIDs> clusters = new Int2ObjectOpenHashMap<>(size); // Allocate working space: MatrixParadigm mat = new MatrixParadigm(ids); ArrayModifiableDBIDs prots = DBIDUtil.newArray(MatrixParadigm.triangleSize(size)); initializeMatrices(mat, prots, dq); DBIDArrayMIter protiter = prots.iter(); FiniteProgress progress = LOG.isVerbose() ? new FiniteProgress("MiniMax clustering", size - 1, LOG) : null; DBIDArrayIter ix = mat.ix; for(int i = 1, end = size; i < size; i++) { end = AGNES.shrinkActiveSet(ix, builder, end, // findMerge(end, mat, protiter, builder, clusters, dq)); LOG.incrementProcessed(progress); } LOG.ensureCompleted(progress); return (PointerPrototypeHierarchyRepresentationResult) builder.complete(); }
/** * Update the entries of the matrices that contain a distance to c, the newly * merged cluster. * * @param size number of ids in the data set * @param mat matrix paradigm * @param prots calculated prototypes * @param builder Result builder * @param clusters the clusters * @param dq distance query of the data set * @param c the cluster to update distances to */ protected static <O> void updateMatrices(int size, MatrixParadigm mat, DBIDArrayMIter prots, PointerHierarchyRepresentationBuilder builder, Int2ObjectOpenHashMap<ModifiableDBIDs> clusters, DistanceQuery<O> dq, int c) { final DBIDArrayIter ix = mat.ix, iy = mat.iy; // c is the new cluster. // Update entries (at (x,y) with x > y) in the matrix where x = c or y = c // Update entries at (c,y) with y < c ix.seek(c); for(iy.seek(0); iy.getOffset() < c; iy.advance()) { // Skip entry if already merged if(builder.isLinked(iy)) { continue; } updateEntry(mat, prots, clusters, dq, c, iy.getOffset()); } // Update entries at (x,c) with x > c iy.seek(c); for(ix.seek(c + 1); ix.valid(); ix.advance()) { // Skip entry if already merged if(builder.isLinked(ix)) { continue; } updateEntry(mat, prots, clusters, dq, ix.getOffset(), c); } }
/** * Run the algorithm * * @param db Database to run on * @param relation Data relation * @return Clustering result */ public PointerPrototypeHierarchyRepresentationResult run(Database db, Relation<O> relation) { DistanceQuery<O> dq = DatabaseUtil.precomputedDistanceQuery(db, relation, getDistanceFunction(), LOG); final DBIDs ids = relation.getDBIDs(); // Initialize space for result: PointerHierarchyRepresentationBuilder builder = new PointerHierarchyRepresentationBuilder(ids, dq.getDistanceFunction().isSquared()); Int2ObjectOpenHashMap<ModifiableDBIDs> clusters = new Int2ObjectOpenHashMap<>(ids.size()); MatrixParadigm mat = new MatrixParadigm(ids); ArrayModifiableDBIDs prots = DBIDUtil.newArray(MatrixParadigm.triangleSize(ids.size())); MiniMax.initializeMatrices(mat, prots, dq); nnChainCore(mat, prots.iter(), dq, builder, clusters); return (PointerPrototypeHierarchyRepresentationResult) builder.complete(); }
public void stopPlayerTracking(EntityPlayerMP player) { PlayerData oldData = playerTracking.get(player); if(oldData == null) return; synchronized(this) { Int2ObjectOpenHashMap<ExcavationRenderEntry> entries = this.get(oldData.dimensionID); if(entries != null && !entries.isEmpty()) { for(ExcavationRenderEntry entry : entries.values()) { if(entry.domainID == oldData.domainID) entry.removeListener(player); } } } playerTracking.remove(player); }
/** * List of lines are generated from a given file. then this list is used to generate a NodeNamerImpl * * @param file * @return INodeNamer * @throws IOException */ public static INodeNamer load(final File file) throws IOException { final int numNodes = countLines(file, Encoding.DEFAULT_CHARSET_NAME); final Int2ObjectOpenHashMap<String> id2Label = new Int2ObjectOpenHashMap<String>(numNodes); final Object2IntOpenHashMap<String> label2Id = new Object2IntOpenHashMap<String>(numNodes); final LineIterator it = FileUtils.lineIterator(file, Encoding.DEFAULT_CHARSET_NAME); try { int curId = 0; while(it.hasNext()) { final String nodeName = it.next(); id2Label.put(curId, nodeName); label2Id.put(nodeName, curId); curId++; } return new NodeNamerImpl(id2Label, label2Id); } finally { it.close(); } }
/** * Constructor * * @param messageValueFactory Factory for creating message values * @param service Service worker * @param config Hadoop configuration */ public LongByteArrayMessageStore( MessageValueFactory<M> messageValueFactory, CentralizedServiceWorker<LongWritable, ?, ?> service, ImmutableClassesGiraphConfiguration<LongWritable, ?, ?> config) { this.messageValueFactory = messageValueFactory; this.service = service; this.config = config; map = new Int2ObjectOpenHashMap<Long2ObjectOpenHashMap<DataInputOutput>>(); for (int partitionId : service.getPartitionStore().getPartitionIds()) { Partition<LongWritable, ?, ?> partition = service.getPartitionStore().getPartition(partitionId); Long2ObjectOpenHashMap<DataInputOutput> partitionMap = new Long2ObjectOpenHashMap<DataInputOutput>( (int) partition.getVertexCount()); map.put(partitionId, partitionMap); } }
/** * Constructor * * @param service Service worker * @param combiner Message combiner */ public IntFloatMessageStore( CentralizedServiceWorker<IntWritable, ?, ?> service, Combiner<IntWritable, FloatWritable> combiner) { this.service = service; this.combiner = combiner; map = new Int2ObjectOpenHashMap<Int2FloatOpenHashMap>(); for (int partitionId : service.getPartitionStore().getPartitionIds()) { Partition<IntWritable, ?, ?> partition = service.getPartitionStore().getPartition(partitionId); Int2FloatOpenHashMap partitionMap = new Int2FloatOpenHashMap((int) partition.getVertexCount()); map.put(partitionId, partitionMap); } }
/** * Constructor * * @param service Service worker * @param combiner Message combiner */ public LongDoubleMessageStore( CentralizedServiceWorker<LongWritable, ?, ?> service, Combiner<LongWritable, DoubleWritable> combiner) { this.service = service; this.combiner = combiner; map = new Int2ObjectOpenHashMap<Long2DoubleOpenHashMap>(); for (int partitionId : service.getPartitionStore().getPartitionIds()) { Partition<LongWritable, ?, ?> partition = service.getPartitionStore().getPartition(partitionId); Long2DoubleOpenHashMap partitionMap = new Long2DoubleOpenHashMap((int) partition.getVertexCount()); map.put(partitionId, partitionMap); } }
/** * Constructor * * @param messageValueFactory Factory for creating message values * @param service Service worker * @param config Hadoop configuration */ public IntByteArrayMessageStore( MessageValueFactory<M> messageValueFactory, CentralizedServiceWorker<IntWritable, ?, ?> service, ImmutableClassesGiraphConfiguration<IntWritable, ?, ?> config) { this.messageValueFactory = messageValueFactory; this.service = service; this.config = config; map = new Int2ObjectOpenHashMap<Int2ObjectOpenHashMap<DataInputOutput>>(); for (int partitionId : service.getPartitionStore().getPartitionIds()) { Partition<IntWritable, ?, ?> partition = service.getPartitionStore().getPartition(partitionId); Int2ObjectOpenHashMap<DataInputOutput> partitionMap = new Int2ObjectOpenHashMap<DataInputOutput>( (int) partition.getVertexCount()); map.put(partitionId, partitionMap); } }
@Override public void readFieldsForPartition(DataInput in, int partitionId) throws IOException { int size = in.readInt(); Int2ObjectOpenHashMap<DataInputOutput> partitionMap = new Int2ObjectOpenHashMap<DataInputOutput>(size); while (size-- > 0) { int vertexId = in.readInt(); DataInputOutput dataInputOutput = config.createMessagesInputOutput(); dataInputOutput.readFields(in); partitionMap.put(vertexId, dataInputOutput); } synchronized (map) { map.put(partitionId, partitionMap); } }
private boolean isClockReady(PartitionKey partKey, int clock) { boolean ready = clock < 0 || context.getClockVectorManager().getPartClock(partKey.getMatrixId(), partKey.getPartitionId()) >= clock; if(!ready) { try { Int2ObjectOpenHashMap<Int2IntOpenHashMap> clocks = context.getMaster().getTaskMatrixClocks(); context.getClockVectorManager().adjustClocks(clocks); } catch (ServiceException e) { LOG.error("Get Clocks from master falied,", e); } ready = clock < 0 || context.getClockVectorManager().getPartClock(partKey.getMatrixId(), partKey.getPartitionId()) >= clock; } return ready; }
/** * * Create a new GetRowsFlowCache. * * @param totalRequestNum total sub-requests number * @param matrixId matrix id * @param rowIndexToPartSizeCache row index to the number of partitions that contain this row map */ public GetRowsFlowCache(int totalRequestNum, int matrixId, Int2IntOpenHashMap rowIndexToPartSizeCache) { super(totalRequestNum); this.matrixId = matrixId; this.rowIndexToPartSizeCache = rowIndexToPartSizeCache; rowsSplitCache = new Int2ObjectOpenHashMap<List<ServerRow>>(); rowsSplitFutures = new ObjectOpenHashSet<Future<List<ServerRow>>>(totalRequestNum); }
public MatrixOpLogCache() { opLogs = new ConcurrentHashMap<>(); messageQueue = new PriorityBlockingQueue<OpLogMessage>(100, new PriorityComparator()); seqIdToMessageMaps = new Int2ObjectOpenHashMap<Int2ObjectAVLTreeMap<OpLogMessage>>(); waitedMessageQueues = new Int2ObjectOpenHashMap<LinkedBlockingQueue<OpLogMessage>>(); flushListeners = new Int2ObjectOpenHashMap<List<OpLogMessage>>(); seqIdGenerator = new AtomicInteger(0); mergingCounters = new Int2IntOpenHashMap(); stopped = new AtomicBoolean(false); messageToFutureMap = new ConcurrentHashMap<OpLogMessage, Future<VoidResult>>(); }
private BitSet getActiveAttributesFromBitSets(BitSet previouslyActiveAttributes, Int2ObjectOpenHashMap<BitSet> attribute2Refs) { BitSet activeAttributes = new BitSet(this.numColumns); for (int attribute = previouslyActiveAttributes.nextSetBit(0); attribute >= 0; attribute = previouslyActiveAttributes.nextSetBit(attribute + 1)) { // All attributes referenced by this attribute are active activeAttributes.or(attribute2Refs.get(attribute)); // This attribute is active if it references any other attribute if (!attribute2Refs.get(attribute).isEmpty()) activeAttributes.set(attribute); } return activeAttributes; }
private BitSet getActiveAttributesFromLists(BitSet previouslyActiveAttributes, Int2ObjectOpenHashMap<IntSingleLinkedList> attribute2Refs) { BitSet activeAttributes = new BitSet(this.numColumns); for (int attribute = previouslyActiveAttributes.nextSetBit(0); attribute >= 0; attribute = previouslyActiveAttributes.nextSetBit(attribute + 1)) { // All attributes referenced by this attribute are active attribute2Refs.get(attribute).setOwnValuesIn(activeAttributes); // This attribute is active if it references any other attribute if (!attribute2Refs.get(attribute).isEmpty()) activeAttributes.set(attribute); } return activeAttributes; }
protected void addToIndex(Int2ObjectOpenHashMap<ArrayList<ElementLhsPair>> level2elements, int level, OpenBitSet lhs) { level2elements.get(level).add(new ElementLhsPair(this, lhs.clone())); if (this.children != null) { for (int childAttr = 0; childAttr < this.numAttributes; childAttr++) { FDTreeElement element = this.children[childAttr]; if (element != null) { lhs.set(childAttr); element.addToIndex(level2elements, level + 1, lhs); lhs.clear(childAttr); } } } }
private BitSet getActiveAttributesFromBitSets(BitSet previouslyActiveAttributes, Int2ObjectOpenHashMap<BitSet> attribute2Refs) { BitSet activeAttributes = new BitSet(getTotalColumnCount(tables)); for (int attribute = previouslyActiveAttributes.nextSetBit(0); attribute >= 0; attribute = previouslyActiveAttributes.nextSetBit(attribute + 1)) { // All attributes referenced by this attribute are active activeAttributes.or(attribute2Refs.get(attribute)); // This attribute is active if it references any other attribute if (!attribute2Refs.get(attribute).isEmpty()) activeAttributes.set(attribute); } return activeAttributes; }
private BitSet getActiveAttributesFromLists(BitSet previouslyActiveAttributes, Int2ObjectOpenHashMap<IntSingleLinkedList> attribute2Refs) { BitSet activeAttributes = new BitSet(getTotalColumnCount(tables)); for (int attribute = previouslyActiveAttributes.nextSetBit(0); attribute >= 0; attribute = previouslyActiveAttributes.nextSetBit(attribute + 1)) { // All attributes referenced by this attribute are active attribute2Refs.get(attribute).setOwnValuesIn(activeAttributes); // This attribute is active if it references any other attribute if (!attribute2Refs.get(attribute).isEmpty()) activeAttributes.set(attribute); } return activeAttributes; }
private void saveCanonicalIds() { Int2ObjectOpenHashMap mappings = canonicalIdHolder.getAllMappings(); for (ObjectIterator<Int2ObjectMap.Entry<?>> i = mappings.int2ObjectEntrySet().fastIterator(); i .hasNext();) { Int2ObjectMap.Entry<?> entry = i.next(); writeCanonicalId(entry.getIntKey(), entry.getValue()); } }
public void close() throws IOException { super.close(); final Int2ObjectOpenHashMap<MappedByteBuffer> layersCache = thrLocLayersCache.get(); if(layersCache != null) { for(final MappedByteBuffer layer : layersCache.values()) { DirectMemUtil.free(layer); } layersCache.clear(); thrLocLayersCache.set(null); } }
public FactorizedModel(@Nonnull RatingInitilizer ratingInitializer, @Nonnegative int factor, float meanRating, @Nonnull RankInitScheme initScheme, int expectedSize) { this.ratingInitializer = ratingInitializer; this.factor = factor; this.initScheme = initScheme; this.minIndex = 0; this.maxIndex = 0; this.meanRating = ratingInitializer.newRating(meanRating); this.users = new Int2ObjectOpenHashMap<Rating[]>(expectedSize); this.items = new Int2ObjectOpenHashMap<Rating[]>(expectedSize); this.userBias = new Int2ObjectOpenHashMap<Rating>(expectedSize); this.itemBias = new Int2ObjectOpenHashMap<Rating>(expectedSize); this.randU = newRandoms(factor, 31L); this.randI = newRandoms(factor, 41L); }
@Override public void configureMix(@Nonnull ModelUpdateHandler handler, boolean cancelMixRequest) { this.handler = handler; this.cancelMixRequest = cancelMixRequest; if (cancelMixRequest) { if (isDenseModel()) { this.mixedRequests_i = new Int2ObjectOpenHashMap<MixedWeight>(327680); } else { this.mixedRequests_o = new Object2ObjectOpenHashMap<Object, MixedWeight>(327680); } } }
public FMIntFeatureMapModel(@Nonnull FMHyperParameters params) { super(params); this._w0 = 0.f; this._w = new Int2FloatOpenHashMap(DEFAULT_MAPSIZE); _w.defaultReturnValue(0.f); this._V = new Int2ObjectOpenHashMap<float[]>(DEFAULT_MAPSIZE); this._minIndex = 0; this._maxIndex = 0; }
void getEvents(ArrayList<byte[]> events, double eventTh, Int2ObjectOpenHashMap<Feature[] > featureMap, boolean min, Attribute att) { // getting events using merge tree for (int tempBin : att.data.keySet()) { Feature[] features = featureMap.get(tempBin); GraphInput tf = functions.get(tempBin); getEvents(events, tf, features, min, eventTh, false); if (min) { att.minThreshold.put(tempBin, new Float(eventTh)); } else { att.maxThreshold.put(tempBin, new Float(eventTh)); } } }
@Override ReusableNodeLongIterator initializeLeftNodeEdgesLongIterator() { return new ChronologicalMultiSegmentIterator<BipartiteGraphSegment>( this, new LeftSegmentEdgeAccessor<BipartiteGraphSegment>( getReaderAccessibleInfo(), new Int2ObjectOpenHashMap<ReusableNodeIntIterator>( getMaxNumSegments()), new Int2ObjectOpenHashMap<ReusableInternalIdToLongIterator>( getMaxNumSegments()) ) ); }
@Override ReusableNodeRandomLongIterator initializeLeftNodeEdgesRandomLongIterator() { return new MultiSegmentRandomIterator<BipartiteGraphSegment>( this, new LeftSegmentRandomEdgeAccessor<BipartiteGraphSegment>( getReaderAccessibleInfo(), new Int2ObjectOpenHashMap<ReusableInternalIdToLongIterator>( getMaxNumSegments()), new Int2ObjectOpenHashMap<ReusableNodeRandomIntIterator>( getMaxNumSegments()) ) ); }
@Override ReusableNodeLongIterator initializeRightNodeEdgesLongIterator() { return new ChronologicalMultiSegmentIterator<BipartiteGraphSegment>( this, new RightSegmentEdgeAccessor( getReaderAccessibleInfo(), new Int2ObjectOpenHashMap<ReusableNodeIntIterator>( getMaxNumSegments()), new Int2ObjectOpenHashMap<ReusableInternalIdToLongIterator>( getMaxNumSegments()) ) ); }
@Override ReusableNodeRandomLongIterator initializeRightNodeEdgesRandomLongIterator() { return new MultiSegmentRandomIterator<BipartiteGraphSegment>( this, new RightSegmentRandomEdgeAccessor( getReaderAccessibleInfo(), new Int2ObjectOpenHashMap<ReusableInternalIdToLongIterator>( getMaxNumSegments()), new Int2ObjectOpenHashMap<ReusableNodeRandomIntIterator>( getMaxNumSegments()) ) ); }
/** * The constructor tries to reserve most of the memory that is needed for the graph. * * @param maxNumSegments is the maximum number of segments to store * @param maxNumEdgesPerSegment is the maximum number of edges a segment will store */ public MultiSegmentReaderAccessibleInfoProvider(int maxNumSegments, int maxNumEdgesPerSegment) { // this is going to swapped out right away in the addNewSegment call this.multiSegmentReaderAccessibleInfo = new MultiSegmentReaderAccessibleInfo<T>( new Int2ObjectOpenHashMap<T>(maxNumSegments), 0, -1); this.maxNumSegments = maxNumSegments; this.maxNumEdgesPerSegment = maxNumEdgesPerSegment; }
@Override ReusableNodeLongIterator initializeLeftNodeEdgesLongIterator() { return new NodeMetadataMultiSegmentIterator( this, new LeftSegmentEdgeAccessor<NodeMetadataLeftIndexedBipartiteGraphSegment>( getReaderAccessibleInfo(), new Int2ObjectOpenHashMap<ReusableNodeIntIterator>(getMaxNumSegments()), new Int2ObjectOpenHashMap<ReusableInternalIdToLongIterator>(getMaxNumSegments()) ) ); }