Java 类it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap 实例源码

/**
 * Get task clocks for all matrices from Master
 * @return task clocks for all matrices from Master
 * @throws ServiceException
 */
public Int2ObjectOpenHashMap<Int2IntOpenHashMap> getTaskMatrixClocks() throws ServiceException {
  GetTaskMatrixClockResponse response = masterProxy.getTaskMatrixClocks(null,
    GetTaskMatrixClockRequest.newBuilder().build());
  Int2ObjectOpenHashMap<Int2IntOpenHashMap> taskIdToMatrixClocksMap = new Int2ObjectOpenHashMap<>(response.getTaskMatrixClocksCount());

  List<TaskMatrixClock> taskMatrixClocks = response.getTaskMatrixClocksList();
  int size = taskMatrixClocks.size();
  int matrixNum;
  for(int i = 0; i < size; i++) {
    Int2IntOpenHashMap matrixIdToClockMap = new Int2IntOpenHashMap(taskMatrixClocks.get(i).getMatrixClocksCount());
    taskIdToMatrixClocksMap.put(taskMatrixClocks.get(i).getTaskId().getTaskIndex(), matrixIdToClockMap);
    List<MatrixClock> matrixClocks = taskMatrixClocks.get(i).getMatrixClocksList();
    matrixNum = matrixClocks.size();
    for(int j = 0; j < matrixNum; j++) {
      matrixIdToClockMap.put(matrixClocks.get(j).getMatrixId(), matrixClocks.get(j).getClock());
    }
  }

  return taskIdToMatrixClocksMap;
}

/**
 * Adjust clock values
 * @param taskToMatrixClocks taskId->(matrixId->clock) map
 */
public void adjustClocks(Int2ObjectOpenHashMap<Int2IntOpenHashMap> taskToMatrixClocks) {
  ObjectIterator<Int2ObjectMap.Entry<Int2IntOpenHashMap>> taskIter =
    taskToMatrixClocks.int2ObjectEntrySet().fastIterator();
  Int2ObjectMap.Entry<Int2IntOpenHashMap> taskEntry = null;
  int taskId = 0;
  Int2IntOpenHashMap matrixIdToClockMap = null;
  ObjectIterator<Int2IntMap.Entry> matrixIter = null;
  Int2IntMap.Entry matrixEntry = null;

  while(taskIter.hasNext()) {
    taskEntry = taskIter.next();
    taskId = taskEntry.getIntKey();
    matrixIdToClockMap = taskEntry.getValue();
    matrixIter = matrixIdToClockMap.int2IntEntrySet().fastIterator();
    while (matrixIter.hasNext()) {
      matrixEntry = matrixIter.next();
      updateClock(matrixEntry.getIntKey(), taskId, matrixEntry.getIntValue());
    }
  }
}

protected void initializeAttributes() throws AlgorithmExecutionException {
    this.numColumns = this.columnNames.size();

    this.attributeId2attributeObject = new Int2ObjectOpenHashMap<Attribute>(this.numColumns);
    this.attributeObjectQueue = new PriorityQueue<Attribute>(this.numColumns);

    for (int table = 0; table < this.tableNames.length; table++) {
        int firstAttribute = this.tableColumnStartIndexes[table];
        int lastAttribute = (table == this.tableNames.length - 1) ? this.numColumns : this.tableColumnStartIndexes[table + 1];

        for (int attribute = firstAttribute; attribute < lastAttribute; attribute++) {
            Attribute spiderAttribute;
            if (this.databaseConnectionGenerator != null)
                spiderAttribute = new Attribute(attribute, this.columnTypes, this.databaseConnectionGenerator, this.inputRowLimit, this.dao, this.tableNames[table], this.columnNames.get(attribute), this.tempFolder);
            else
                spiderAttribute = new Attribute(attribute, this.columnTypes, this.fileInputGenerator[table], this.inputRowLimit, attribute - firstAttribute, this.tempFolder, this.maxMemoryUsage, this.memoryCheckFrequency);
            this.attributeId2attributeObject.put(attribute, spiderAttribute);

            if (!spiderAttribute.hasFinished())
                this.attributeObjectQueue.add(spiderAttribute);
        }
    }
}

private void output() throws CouldNotReceiveResultException, ColumnNameMismatchException {
    // Read the discovered INDs from the attributes
    Int2ObjectOpenHashMap<IntList> dep2ref = new Int2ObjectOpenHashMap<IntList>(this.numColumns);
    for (Attribute spiderAttribute : this.attributeId2attributeObject.values())
        if (!spiderAttribute.getReferenced().isEmpty())
            dep2ref.put(spiderAttribute.getAttributeId(), new IntArrayList(spiderAttribute.getReferenced()));

    // Write the result to the resultReceiver
    for (int dep : dep2ref.keySet()) {
        String depTableName = this.getTableNameFor(dep, this.tableColumnStartIndexes);
        String depColumnName = this.columnNames.get(dep);

        for (int ref : dep2ref.get(dep)) {
            String refTableName = this.getTableNameFor(ref, this.tableColumnStartIndexes);
            String refColumnName = this.columnNames.get(ref);

            this.resultReceiver.receiveResult(new InclusionDependency(new ColumnPermutation(new ColumnIdentifier(depTableName, depColumnName)), new ColumnPermutation(new ColumnIdentifier(refTableName, refColumnName))));
            this.numUnaryINDs++;
        }
    }
}

public void generalize() {
    int maxLevel = this.numAttributes;

    // Build an index level->nodes for the top-down, level-wise traversal
    Int2ObjectOpenHashMap<ArrayList<ElementLhsPair>> level2elements = new Int2ObjectOpenHashMap<>(maxLevel);
    for (int level = 0; level < maxLevel; level++)
        level2elements.put(level, new ArrayList<ElementLhsPair>());
    this.addToIndex(level2elements, 0, new OpenBitSet(this.numAttributes));

    // Traverse the levels top-down and add all direct generalizations
    for (int level = maxLevel - 1; level >= 0; level--) {
        for (ElementLhsPair pair : level2elements.get(level)) {
            // Remove isFDs, because we will mark valid FDs later on
            pair.element.removeAllFds();

            // Generate and add generalizations
            for (int lhsAttr = pair.lhs.nextSetBit(0); lhsAttr >= 0; lhsAttr = pair.lhs.nextSetBit(lhsAttr + 1)) {
                pair.lhs.clear(lhsAttr);
                FDTreeElement generalization = this.addGeneralization(pair.lhs, pair.element.getRhsAttributes());
                if (generalization != null)
                    level2elements.get(level - 1).add(new ElementLhsPair(generalization, pair.lhs.clone()));
                pair.lhs.set(lhsAttr);
            }
        }
    }
}

protected static Entry<Map<String, SimpleEnum<?>>, Int2ObjectMap<SimpleEnum<?>>> init(final Class<?> clazz, final int size)
{
    Map<String, SimpleEnum<?>> byName = ASimpleEnum.byName.get(clazz);
    if (byName == null)
    {
        byName = new CaseInsensitiveMap<>(size, SMALL_LOAD_FACTOR);
        ASimpleEnum.byName.put(clazz, byName);
    }
    Int2ObjectMap<SimpleEnum<?>> byID = ASimpleEnum.byOrdinal.get(clazz);
    if (byID == null)
    {
        byID = new Int2ObjectOpenHashMap<>(size, SMALL_LOAD_FACTOR);
        ASimpleEnum.byOrdinal.put(clazz, byID);
    }
    return new SimpleEntry<>(byName, byID);
}

private void replayTrain(@Nonnull final ByteBuffer buf) {
    final int itemI = buf.getInt();
    final int knnSize = buf.getInt();

    final Int2ObjectMap<Int2FloatMap> knnItems = new Int2ObjectOpenHashMap<>(1024);
    final IntSet pairItems = new IntOpenHashSet();
    for (int i = 0; i < knnSize; i++) {
        int user = buf.getInt();
        int ruSize = buf.getInt();
        Int2FloatMap ru = new Int2FloatOpenHashMap(ruSize);
        ru.defaultReturnValue(0.f);

        for (int j = 0; j < ruSize; j++) {
            int itemK = buf.getInt();
            pairItems.add(itemK);
            float ruk = buf.getFloat();
            ru.put(itemK, ruk);
        }
        knnItems.put(user, ru);
    }

    for (int itemJ : pairItems) {
        train(itemI, knnItems, itemJ);
    }
}

@Nonnull
private static Int2ObjectMap<Int2FloatMap> kNNentries(@Nonnull final Object kNNiObj,
        @Nonnull final MapObjectInspector knnItemsOI,
        @Nonnull final PrimitiveObjectInspector knnItemsKeyOI,
        @Nonnull final MapObjectInspector knnItemsValueOI,
        @Nonnull final PrimitiveObjectInspector knnItemsValueKeyOI,
        @Nonnull final PrimitiveObjectInspector knnItemsValueValueOI,
        @Nullable Int2ObjectMap<Int2FloatMap> knnItems, @Nonnull final MutableInt nnzKNNi) {
    if (knnItems == null) {
        knnItems = new Int2ObjectOpenHashMap<>(1024);
    } else {
        knnItems.clear();
    }

    int numElementOfKNNItems = 0;
    for (Map.Entry<?, ?> entry : knnItemsOI.getMap(kNNiObj).entrySet()) {
        int user = PrimitiveObjectInspectorUtils.getInt(entry.getKey(), knnItemsKeyOI);
        Int2FloatMap ru = int2floatMap(knnItemsValueOI.getMap(entry.getValue()),
            knnItemsValueKeyOI, knnItemsValueValueOI);
        knnItems.put(user, ru);
        numElementOfKNNItems += ru.size();
    }

    nnzKNNi.setValue(numElementOfKNNItems);
    return knnItems;
}

protected void calculateLayerArrays() {
    Int2ObjectOpenHashMap<char[]> colorLayerMap = new Int2ObjectOpenHashMap<>();
    for (int i = 0; i < validBlockIds.length; i++) {
        int color = validColors[i];
        int combined = validBlockIds[i];
        if (hasAlpha(color)) {
            for (int j = 0; j < validBlockIds.length; j++) {
                int colorOther = validColors[j];
                if (!hasAlpha(colorOther)) {
                    int combinedOther = validBlockIds[j];
                    int combinedColor = combineTransparency(color, colorOther);
                    colorLayerMap.put(combinedColor, new char[]{(char) combined, (char) combinedOther});
                }
            }
        }
    }
    this.validLayerColors = new int[colorLayerMap.size()];
    this.validLayerBlocks = new char[colorLayerMap.size()][];
    int index = 0;
    for (Int2ObjectMap.Entry<char[]> entry : colorLayerMap.int2ObjectEntrySet()) {
        validLayerColors[index] = entry.getIntKey();
        validLayerBlocks[index++] = entry.getValue();
    }
}

/**
 * Get the candidates: points which have at least one hash bucket in common.
 * 
 * @param obj Query object
 * @return Candidates
 */
protected DBIDs getCandidates(V obj) {
  ModifiableDBIDs candidates = null;
  final int numhash = hashtables.size();
  double[] buf = new double[hashfunctions.get(0).getNumberOfProjections()];
  for(int i = 0; i < numhash; i++) {
    final Int2ObjectOpenHashMap<DBIDs> table = hashtables.get(i);
    final LocalitySensitiveHashFunction<? super V> hashfunc = hashfunctions.get(i);
    // Get the initial (unbounded) hash code:
    int hash = hashfunc.hashObject(obj, buf);
    // Reduce to hash table size
    int bucket = hash % numberOfBuckets;
    DBIDs cur = table.get(bucket);
    if(cur != null) {
      if(candidates == null) {
        candidates = DBIDUtil.newHashSet(cur.size() * numhash);
      }
      candidates.addDBIDs(cur);
    }
  }
  return (candidates == null) ? DBIDUtil.EMPTYDBIDS : candidates;
}

/**
 * Run the algorithm on a database.
 * 
 * @param db Database
 * @param relation Relation to process.
 * @return Hierarchical result
 */
public PointerPrototypeHierarchyRepresentationResult run(Database db, Relation<O> relation) {
  DistanceQuery<O> dq = DatabaseUtil.precomputedDistanceQuery(db, relation, getDistanceFunction(), LOG);
  final DBIDs ids = relation.getDBIDs();
  final int size = ids.size();

  // Initialize space for result:
  PointerHierarchyRepresentationBuilder builder = new PointerHierarchyRepresentationBuilder(ids, dq.getDistanceFunction().isSquared());
  Int2ObjectOpenHashMap<ModifiableDBIDs> clusters = new Int2ObjectOpenHashMap<>(size);

  // Allocate working space:
  MatrixParadigm mat = new MatrixParadigm(ids);
  ArrayModifiableDBIDs prots = DBIDUtil.newArray(MatrixParadigm.triangleSize(size));
  initializeMatrices(mat, prots, dq);

  DBIDArrayMIter protiter = prots.iter();
  FiniteProgress progress = LOG.isVerbose() ? new FiniteProgress("MiniMax clustering", size - 1, LOG) : null;
  DBIDArrayIter ix = mat.ix;
  for(int i = 1, end = size; i < size; i++) {
    end = AGNES.shrinkActiveSet(ix, builder, end, //
        findMerge(end, mat, protiter, builder, clusters, dq));
    LOG.incrementProcessed(progress);
  }
  LOG.ensureCompleted(progress);
  return (PointerPrototypeHierarchyRepresentationResult) builder.complete();
}

/**
 * Update the entries of the matrices that contain a distance to c, the newly
 * merged cluster.
 * 
 * @param size number of ids in the data set
 * @param mat matrix paradigm
 * @param prots calculated prototypes
 * @param builder Result builder
 * @param clusters the clusters
 * @param dq distance query of the data set
 * @param c the cluster to update distances to
 */
protected static <O> void updateMatrices(int size, MatrixParadigm mat, DBIDArrayMIter prots, PointerHierarchyRepresentationBuilder builder, Int2ObjectOpenHashMap<ModifiableDBIDs> clusters, DistanceQuery<O> dq, int c) {
  final DBIDArrayIter ix = mat.ix, iy = mat.iy;
  // c is the new cluster.
  // Update entries (at (x,y) with x > y) in the matrix where x = c or y = c

  // Update entries at (c,y) with y < c
  ix.seek(c);
  for(iy.seek(0); iy.getOffset() < c; iy.advance()) {
    // Skip entry if already merged
    if(builder.isLinked(iy)) {
      continue;
    }
    updateEntry(mat, prots, clusters, dq, c, iy.getOffset());
  }

  // Update entries at (x,c) with x > c
  iy.seek(c);
  for(ix.seek(c + 1); ix.valid(); ix.advance()) {
    // Skip entry if already merged
    if(builder.isLinked(ix)) {
      continue;
    }
    updateEntry(mat, prots, clusters, dq, ix.getOffset(), c);
  }
}

/**
 * Run the algorithm
 * 
 * @param db Database to run on
 * @param relation Data relation
 * @return Clustering result
 */
public PointerPrototypeHierarchyRepresentationResult run(Database db, Relation<O> relation) {
  DistanceQuery<O> dq = DatabaseUtil.precomputedDistanceQuery(db, relation, getDistanceFunction(), LOG);
  final DBIDs ids = relation.getDBIDs();

  // Initialize space for result:
  PointerHierarchyRepresentationBuilder builder = new PointerHierarchyRepresentationBuilder(ids, dq.getDistanceFunction().isSquared());
  Int2ObjectOpenHashMap<ModifiableDBIDs> clusters = new Int2ObjectOpenHashMap<>(ids.size());

  MatrixParadigm mat = new MatrixParadigm(ids);
  ArrayModifiableDBIDs prots = DBIDUtil.newArray(MatrixParadigm.triangleSize(ids.size()));

  MiniMax.initializeMatrices(mat, prots, dq);

  nnChainCore(mat, prots.iter(), dq, builder, clusters);

  return (PointerPrototypeHierarchyRepresentationResult) builder.complete();
}

public void stopPlayerTracking(EntityPlayerMP player)
{
    PlayerData oldData = playerTracking.get(player);

    if(oldData == null) return;

    synchronized(this)
    {
        Int2ObjectOpenHashMap<ExcavationRenderEntry> entries = this.get(oldData.dimensionID);
        if(entries != null && !entries.isEmpty())
        {
            for(ExcavationRenderEntry entry : entries.values())
            {
                if(entry.domainID == oldData.domainID) entry.removeListener(player);
            }
        }
    }

    playerTracking.remove(player);
}

/**
 * List of lines are generated from a given file. then this list is used to generate a NodeNamerImpl
 *
 * @param file
 * @return INodeNamer
 * @throws IOException
 */
public static INodeNamer load(final File file) throws IOException {
    final int numNodes = countLines(file, Encoding.DEFAULT_CHARSET_NAME);

    final Int2ObjectOpenHashMap<String> id2Label = new Int2ObjectOpenHashMap<String>(numNodes);
    final Object2IntOpenHashMap<String> label2Id = new Object2IntOpenHashMap<String>(numNodes);

    final LineIterator it = FileUtils.lineIterator(file, Encoding.DEFAULT_CHARSET_NAME);
    try {
        int curId = 0;
        while(it.hasNext()) {
            final String nodeName = it.next();
            id2Label.put(curId, nodeName);
            label2Id.put(nodeName, curId);
            curId++;
        }

        return new NodeNamerImpl(id2Label, label2Id);
    } finally {
        it.close();
    }
}

/**
 * Constructor
 *
 * @param messageValueFactory Factory for creating message values
 * @param service      Service worker
 * @param config       Hadoop configuration
 */
public LongByteArrayMessageStore(
    MessageValueFactory<M> messageValueFactory,
    CentralizedServiceWorker<LongWritable, ?, ?> service,
    ImmutableClassesGiraphConfiguration<LongWritable, ?, ?> config) {
  this.messageValueFactory = messageValueFactory;
  this.service = service;
  this.config = config;

  map =
      new Int2ObjectOpenHashMap<Long2ObjectOpenHashMap<DataInputOutput>>();
  for (int partitionId : service.getPartitionStore().getPartitionIds()) {
    Partition<LongWritable, ?, ?> partition =
        service.getPartitionStore().getPartition(partitionId);
    Long2ObjectOpenHashMap<DataInputOutput> partitionMap =
        new Long2ObjectOpenHashMap<DataInputOutput>(
            (int) partition.getVertexCount());
    map.put(partitionId, partitionMap);
  }
}

/**
 * Constructor
 *
 * @param service Service worker
 * @param combiner Message combiner
 */
public IntFloatMessageStore(
    CentralizedServiceWorker<IntWritable, ?, ?> service,
    Combiner<IntWritable, FloatWritable> combiner) {
  this.service = service;
  this.combiner = combiner;

  map = new Int2ObjectOpenHashMap<Int2FloatOpenHashMap>();
  for (int partitionId : service.getPartitionStore().getPartitionIds()) {
    Partition<IntWritable, ?, ?> partition =
        service.getPartitionStore().getPartition(partitionId);
    Int2FloatOpenHashMap partitionMap =
        new Int2FloatOpenHashMap((int) partition.getVertexCount());
    map.put(partitionId, partitionMap);
  }
}

/**
 * Constructor
 *
 * @param service Service worker
 * @param combiner Message combiner
 */
public LongDoubleMessageStore(
    CentralizedServiceWorker<LongWritable, ?, ?> service,
    Combiner<LongWritable, DoubleWritable> combiner) {
  this.service = service;
  this.combiner = combiner;

  map = new Int2ObjectOpenHashMap<Long2DoubleOpenHashMap>();
  for (int partitionId : service.getPartitionStore().getPartitionIds()) {
    Partition<LongWritable, ?, ?> partition =
        service.getPartitionStore().getPartition(partitionId);
    Long2DoubleOpenHashMap partitionMap =
        new Long2DoubleOpenHashMap((int) partition.getVertexCount());
    map.put(partitionId, partitionMap);
  }
}

/**
 * Constructor
 *
 * @param messageValueFactory Factory for creating message values
 * @param service      Service worker
 * @param config       Hadoop configuration
 */
public IntByteArrayMessageStore(
    MessageValueFactory<M> messageValueFactory,
    CentralizedServiceWorker<IntWritable, ?, ?> service,
    ImmutableClassesGiraphConfiguration<IntWritable, ?, ?> config) {
  this.messageValueFactory = messageValueFactory;
  this.service = service;
  this.config = config;

  map =
      new Int2ObjectOpenHashMap<Int2ObjectOpenHashMap<DataInputOutput>>();
  for (int partitionId : service.getPartitionStore().getPartitionIds()) {
    Partition<IntWritable, ?, ?> partition =
        service.getPartitionStore().getPartition(partitionId);
    Int2ObjectOpenHashMap<DataInputOutput> partitionMap =
        new Int2ObjectOpenHashMap<DataInputOutput>(
            (int) partition.getVertexCount());
    map.put(partitionId, partitionMap);
  }
}

@Override
public void readFieldsForPartition(DataInput in,
    int partitionId) throws IOException {
  int size = in.readInt();
  Int2ObjectOpenHashMap<DataInputOutput> partitionMap =
      new Int2ObjectOpenHashMap<DataInputOutput>(size);
  while (size-- > 0) {
    int vertexId = in.readInt();
    DataInputOutput dataInputOutput = config.createMessagesInputOutput();
    dataInputOutput.readFields(in);
    partitionMap.put(vertexId, dataInputOutput);
  }
  synchronized (map) {
    map.put(partitionId, partitionMap);
  }
}

private boolean isClockReady(PartitionKey partKey, int clock) {
  boolean ready = clock < 0 || context.getClockVectorManager().getPartClock(partKey.getMatrixId(), partKey.getPartitionId()) >= clock;
  if(!ready) {
    try {
      Int2ObjectOpenHashMap<Int2IntOpenHashMap> clocks = context.getMaster().getTaskMatrixClocks();
      context.getClockVectorManager().adjustClocks(clocks);
    } catch (ServiceException e) {
      LOG.error("Get Clocks from master falied,", e);
    }
    ready = clock < 0 || context.getClockVectorManager().getPartClock(partKey.getMatrixId(), partKey.getPartitionId()) >= clock;
  }
  return ready;
}

/**
 * 
 * Create a new GetRowsFlowCache.
 *
 * @param totalRequestNum total sub-requests number
 * @param matrixId matrix id
 * @param rowIndexToPartSizeCache row index to the number of partitions that contain this row map
 */
public GetRowsFlowCache(int totalRequestNum, int matrixId,
    Int2IntOpenHashMap rowIndexToPartSizeCache) {
  super(totalRequestNum);
  this.matrixId = matrixId;
  this.rowIndexToPartSizeCache = rowIndexToPartSizeCache;
  rowsSplitCache = new Int2ObjectOpenHashMap<List<ServerRow>>();
  rowsSplitFutures = new ObjectOpenHashSet<Future<List<ServerRow>>>(totalRequestNum);
}

public MatrixOpLogCache() {
  opLogs = new ConcurrentHashMap<>();

  messageQueue = new PriorityBlockingQueue<OpLogMessage>(100, new PriorityComparator());
  seqIdToMessageMaps = new Int2ObjectOpenHashMap<Int2ObjectAVLTreeMap<OpLogMessage>>();
  waitedMessageQueues = new Int2ObjectOpenHashMap<LinkedBlockingQueue<OpLogMessage>>();
  flushListeners = new Int2ObjectOpenHashMap<List<OpLogMessage>>();
  seqIdGenerator = new AtomicInteger(0);
  mergingCounters = new Int2IntOpenHashMap();
  stopped = new AtomicBoolean(false);
  messageToFutureMap = new ConcurrentHashMap<OpLogMessage, Future<VoidResult>>();
}

private BitSet getActiveAttributesFromBitSets(BitSet previouslyActiveAttributes, Int2ObjectOpenHashMap<BitSet> attribute2Refs) {
    BitSet activeAttributes = new BitSet(this.numColumns);
    for (int attribute = previouslyActiveAttributes.nextSetBit(0); attribute >= 0; attribute = previouslyActiveAttributes.nextSetBit(attribute + 1)) {
        // All attributes referenced by this attribute are active
        activeAttributes.or(attribute2Refs.get(attribute));
        // This attribute is active if it references any other attribute
        if (!attribute2Refs.get(attribute).isEmpty())
            activeAttributes.set(attribute);
    }
    return activeAttributes;
}

private BitSet getActiveAttributesFromLists(BitSet previouslyActiveAttributes, Int2ObjectOpenHashMap<IntSingleLinkedList> attribute2Refs) {
    BitSet activeAttributes = new BitSet(this.numColumns);
    for (int attribute = previouslyActiveAttributes.nextSetBit(0); attribute >= 0; attribute = previouslyActiveAttributes.nextSetBit(attribute + 1)) {
        // All attributes referenced by this attribute are active
        attribute2Refs.get(attribute).setOwnValuesIn(activeAttributes);
        // This attribute is active if it references any other attribute
        if (!attribute2Refs.get(attribute).isEmpty())
            activeAttributes.set(attribute);
    }
    return activeAttributes;
}

protected void addToIndex(Int2ObjectOpenHashMap<ArrayList<ElementLhsPair>> level2elements, int level, OpenBitSet lhs) {
    level2elements.get(level).add(new ElementLhsPair(this, lhs.clone()));
    if (this.children != null) {
        for (int childAttr = 0; childAttr < this.numAttributes; childAttr++) {
            FDTreeElement element = this.children[childAttr];
            if (element != null) {
                lhs.set(childAttr);
                element.addToIndex(level2elements, level + 1, lhs);
                lhs.clear(childAttr);
            }
        }
    }
}

private BitSet getActiveAttributesFromBitSets(BitSet previouslyActiveAttributes, Int2ObjectOpenHashMap<BitSet> attribute2Refs) {
    BitSet activeAttributes = new BitSet(getTotalColumnCount(tables));
    for (int attribute = previouslyActiveAttributes.nextSetBit(0); attribute >= 0; attribute = previouslyActiveAttributes.nextSetBit(attribute + 1)) {
        // All attributes referenced by this attribute are active
        activeAttributes.or(attribute2Refs.get(attribute));
        // This attribute is active if it references any other attribute
        if (!attribute2Refs.get(attribute).isEmpty())
            activeAttributes.set(attribute);
    }
    return activeAttributes;
}

private BitSet getActiveAttributesFromLists(BitSet previouslyActiveAttributes, Int2ObjectOpenHashMap<IntSingleLinkedList> attribute2Refs) {
    BitSet activeAttributes = new BitSet(getTotalColumnCount(tables));
    for (int attribute = previouslyActiveAttributes.nextSetBit(0); attribute >= 0; attribute = previouslyActiveAttributes.nextSetBit(attribute + 1)) {
        // All attributes referenced by this attribute are active
        attribute2Refs.get(attribute).setOwnValuesIn(activeAttributes);
        // This attribute is active if it references any other attribute
        if (!attribute2Refs.get(attribute).isEmpty())
            activeAttributes.set(attribute);
    }
    return activeAttributes;
}

private void saveCanonicalIds() {
  Int2ObjectOpenHashMap mappings = canonicalIdHolder.getAllMappings();
  for (ObjectIterator<Int2ObjectMap.Entry<?>> i = mappings.int2ObjectEntrySet().fastIterator(); i
      .hasNext();) {
    Int2ObjectMap.Entry<?> entry = i.next();
    writeCanonicalId(entry.getIntKey(), entry.getValue());
  }
}

public void close()
throws IOException {
    super.close();
    final Int2ObjectOpenHashMap<MappedByteBuffer>
        layersCache = thrLocLayersCache.get();
    if(layersCache != null) {
        for(final MappedByteBuffer layer : layersCache.values()) {
            DirectMemUtil.free(layer);
        }
        layersCache.clear();
        thrLocLayersCache.set(null);
    }
}

public FactorizedModel(@Nonnull RatingInitilizer ratingInitializer, @Nonnegative int factor,
        float meanRating, @Nonnull RankInitScheme initScheme, int expectedSize) {
    this.ratingInitializer = ratingInitializer;
    this.factor = factor;
    this.initScheme = initScheme;
    this.minIndex = 0;
    this.maxIndex = 0;
    this.meanRating = ratingInitializer.newRating(meanRating);
    this.users = new Int2ObjectOpenHashMap<Rating[]>(expectedSize);
    this.items = new Int2ObjectOpenHashMap<Rating[]>(expectedSize);
    this.userBias = new Int2ObjectOpenHashMap<Rating>(expectedSize);
    this.itemBias = new Int2ObjectOpenHashMap<Rating>(expectedSize);
    this.randU = newRandoms(factor, 31L);
    this.randI = newRandoms(factor, 41L);
}

@Override
public void configureMix(@Nonnull ModelUpdateHandler handler, boolean cancelMixRequest) {
    this.handler = handler;
    this.cancelMixRequest = cancelMixRequest;
    if (cancelMixRequest) {
        if (isDenseModel()) {
            this.mixedRequests_i = new Int2ObjectOpenHashMap<MixedWeight>(327680);
        } else {
            this.mixedRequests_o = new Object2ObjectOpenHashMap<Object, MixedWeight>(327680);
        }
    }
}

public FMIntFeatureMapModel(@Nonnull FMHyperParameters params) {
    super(params);
    this._w0 = 0.f;
    this._w = new Int2FloatOpenHashMap(DEFAULT_MAPSIZE);
    _w.defaultReturnValue(0.f);
    this._V = new Int2ObjectOpenHashMap<float[]>(DEFAULT_MAPSIZE);
    this._minIndex = 0;
    this._maxIndex = 0;
}

void getEvents(ArrayList<byte[]> events, double eventTh,
        Int2ObjectOpenHashMap<Feature[] > featureMap, boolean min, Attribute att) {
    // getting events using merge tree
    for (int tempBin : att.data.keySet()) {
        Feature[] features = featureMap.get(tempBin);
        GraphInput tf = functions.get(tempBin);
        getEvents(events, tf, features, min, eventTh, false);

        if (min) {
               att.minThreshold.put(tempBin, new Float(eventTh));
           } else {
               att.maxThreshold.put(tempBin, new Float(eventTh));
           }
    }
}

@Override
ReusableNodeLongIterator initializeLeftNodeEdgesLongIterator() {
  return new ChronologicalMultiSegmentIterator<BipartiteGraphSegment>(
      this,
      new LeftSegmentEdgeAccessor<BipartiteGraphSegment>(
          getReaderAccessibleInfo(),
          new Int2ObjectOpenHashMap<ReusableNodeIntIterator>(
              getMaxNumSegments()),
          new Int2ObjectOpenHashMap<ReusableInternalIdToLongIterator>(
              getMaxNumSegments())
      )
  );
}

@Override
ReusableNodeRandomLongIterator initializeLeftNodeEdgesRandomLongIterator() {
  return new MultiSegmentRandomIterator<BipartiteGraphSegment>(
      this,
      new LeftSegmentRandomEdgeAccessor<BipartiteGraphSegment>(
          getReaderAccessibleInfo(),
          new Int2ObjectOpenHashMap<ReusableInternalIdToLongIterator>(
              getMaxNumSegments()),
          new Int2ObjectOpenHashMap<ReusableNodeRandomIntIterator>(
              getMaxNumSegments())
      )
  );
}

@Override
ReusableNodeLongIterator initializeRightNodeEdgesLongIterator() {
  return new ChronologicalMultiSegmentIterator<BipartiteGraphSegment>(
      this,
      new RightSegmentEdgeAccessor(
          getReaderAccessibleInfo(),
          new Int2ObjectOpenHashMap<ReusableNodeIntIterator>(
              getMaxNumSegments()),
          new Int2ObjectOpenHashMap<ReusableInternalIdToLongIterator>(
              getMaxNumSegments())
      )
  );
}

@Override
ReusableNodeRandomLongIterator initializeRightNodeEdgesRandomLongIterator() {
  return new MultiSegmentRandomIterator<BipartiteGraphSegment>(
      this,
      new RightSegmentRandomEdgeAccessor(
          getReaderAccessibleInfo(),
          new Int2ObjectOpenHashMap<ReusableInternalIdToLongIterator>(
              getMaxNumSegments()),
          new Int2ObjectOpenHashMap<ReusableNodeRandomIntIterator>(
              getMaxNumSegments())
      )
  );
}

/**
 * The constructor tries to reserve most of the memory that is needed for the graph.
 *
 * @param maxNumSegments         is the maximum number of segments to store
 * @param maxNumEdgesPerSegment  is the maximum number of edges a segment will store
 */
public MultiSegmentReaderAccessibleInfoProvider(int maxNumSegments, int maxNumEdgesPerSegment) {
  // this is going to swapped out right away in the addNewSegment call
  this.multiSegmentReaderAccessibleInfo = new MultiSegmentReaderAccessibleInfo<T>(
      new Int2ObjectOpenHashMap<T>(maxNumSegments),
      0,
      -1);
  this.maxNumSegments = maxNumSegments;
  this.maxNumEdgesPerSegment = maxNumEdgesPerSegment;
}

@Override
ReusableNodeLongIterator initializeLeftNodeEdgesLongIterator() {
  return new NodeMetadataMultiSegmentIterator(
    this,
    new LeftSegmentEdgeAccessor<NodeMetadataLeftIndexedBipartiteGraphSegment>(
      getReaderAccessibleInfo(),
      new Int2ObjectOpenHashMap<ReusableNodeIntIterator>(getMaxNumSegments()),
      new Int2ObjectOpenHashMap<ReusableInternalIdToLongIterator>(getMaxNumSegments())
    )
  );
}