Java 类it.unimi.dsi.fastutil.ints.Int2LongOpenHashMap 实例源码

public void phase3(int id) {
    Map<Integer, Int2LongMap> h = JCL_FacadeImpl.GetHashMap(String.valueOf(id));
    Int2LongMap result = new Int2LongOpenHashMap();
    for (Int2LongMap m : h.values()) {
        for (int i : m.keySet()) {
            if (result.containsKey(i)) {
                long j = m.get(i) + result.get(i);
                result.put(i, j);
            } else {
                result.put(i, m.get(i));
            }
        }
        m.clear();
    }
    long freqT = 0;
    for (Long v : result.values())
        freqT += v;
    System.err.println("ID: " + id + " size: " + result.size() + " freqT: " + freqT);

    h.clear();
    h.put(id, result);
    // result.clear();
    // result = null;
}

/**
 * Find the parent entity for each.
 *
 * @param parents Parent counter map
 * @param parent Parent storage (output)
 * @param cnt Count storage (output)
 */
private void findParent(Int2LongOpenHashMap[] parents, int[] parent, long[] cnt) {
  parent[0] = 0;
  cnt[0] = Long.MAX_VALUE;
  for(int i = 1; i < parents.length; i++) {
    int best = i;
    long total = 0, bcount = -1;
    for(Int2LongMap.Entry p : parents[i].int2LongEntrySet()) {
      final long c = p.getLongValue();
      total += c;
      if(c > bcount || (c == bcount && c < best)) {
        bcount = c;
        best = p.getIntKey();
      }
    }
    cnt[i] = total;
    parent[i] = best;
  }
}

public FFMStringFeatureMapModel(@Nonnull FFMHyperParameters params) {
    super(params);
    this._w0 = 0.f;
    this._map = new Int2LongOpenHashMap(DEFAULT_MAPSIZE);
    _map.defaultReturnValue(-1L);
    this._buf = new HeapBuffer(HeapBuffer.DEFAULT_CHUNK_SIZE);
    this._freelistW = new LongArrayList();
    this._freelistV = new LongArrayList();
    this._initV = true;
    this._removedV = new RoaringBitmap();
    this._numFields = params.numFields;
    this._entrySizeW = entrySize(1, _useFTRL, _useAdaGrad);
    this._entrySizeV = entrySize(_factor, _useFTRL, _useAdaGrad);
}

private String buildInputDataPartitionSchema(List<JCL_result> r, int numOfJCLThreads){

    IntSet sorted = new IntAVLTreeSet();
    long totalF=0;
    Int2LongMap map = new Int2LongOpenHashMap();

    for(JCL_result oneR:r){

        try{
            @SuppressWarnings("unchecked")
            List<String> l = (List<String>) oneR.getCorrectResult();

            for(String s : l){
                String[] args = s.split(":");
                int key = Integer.parseInt(args[0]); long freq = Long.parseLong(args[1]);
                sorted.add(key);

                if(map.containsKey(key)){
                    freq+=map.get(key); 
                    totalF+=map.get(key);
                } else totalF+=freq;

                map.put(key, freq);
            }

        }catch(Exception e){}
    }

    long load=0; int b; String result = "";
    for(int ac:sorted){
        load += map.get(ac);
        if(load > (totalF/(numOfJCLThreads))){                  
            b=ac;
            result += b + ":";              
            load=0;
        }   
    }       

    return result;
}

/**
 * Flatten another layers onto a slice.
 *
 * @param slice Output slice
 * @param layer Input layer
 * @param parents Parents map
 * @param meta Reduce metadata
 * @param viewport Viewport
 */
private void flatten(int[][] slice, int[][] layer, Int2LongOpenHashMap[] parents, Viewport viewport) {
  // Count the most frequent parent for each entity.
  for(int y = 0; y < viewport.height; y++) {
    final int[] rowy = layer[y], outy = slice[y];
    for(int x = 0; x < viewport.width; x++) {
      int id = rowy[x] & 0x00FF_FFFF; // top byte is alpha!
      if(id == 0) {
        continue;
      }
      parents[id].addTo(outy[x], 1);
      outy[x] = id;
    }
  }
}

void createSplitsGreedy(Map<String, Set<OneBlockInfo>> nodeToBlocks,
  Map<OneBlockInfo, String[]> blockToNodes,
  Map<String, List<OneBlockInfo>> rackToBlocks,
  long totLength,
  int num,
  long minSizeNode,
  long minSizeRack,
  List<InputSplit> splits) {

  List<OneBlockInfo> blocks = new ArrayList<>(blockToNodes.keySet());
  Collections.sort(blocks, new Comparator<OneBlockInfo>() {
    @Override
    public int compare(OneBlockInfo o1, OneBlockInfo o2) {
      return -(int) (o1.length - o2.length);
    }
  });

  Int2LongOpenHashMap loads = new Int2LongOpenHashMap();
  Map<Integer, ArrayList<OneBlockInfo>> parts = new HashMap<>();
  Map<Integer, Set<String>> locations = new HashMap<>();

  //    long num = totLength / maxSize;

  for (int i = 0; i < num; i ++) {
    parts.put(i, new ArrayList<OneBlockInfo>());
    locations.put(i, new HashSet<String>());
  }

  for (OneBlockInfo blockInfo : blocks) {

    long min = Long.MAX_VALUE;
    int selectPart = -1;
    for (int s = 0; s < num; s ++) {
      if (loads.get(s) < min) {
        min = loads.get(s);
        selectPart = s;
      }
    }

    loads.addTo(selectPart, blockInfo.length);
    parts.get(selectPart).add(blockInfo);
    for (String host : blockInfo.hosts)
      locations.get(selectPart).add(host);
  }

  for (Map.Entry<Integer, ArrayList<OneBlockInfo>> entry: parts.entrySet()) {
    addCreatedSplit(splits, locations.get(entry.getKey()), entry.getValue());
  }
}

public List<String> phase1(int id, String name, int numJCLThreads) {
    Int2LongMap values = new Int2LongOpenHashMap(1000000);
    long totalF = 0;
    System.err.println("file: " + name);
    try {
        File f = new File("../" + name + "/" + name + ".bin");
        InputStream in = new BufferedInputStream(new FileInputStream(f));
        FastBufferedInputStream fb = new FastBufferedInputStream(in);
        byte[] i = new byte[4];
        while (fb.read(i) == 4) {
            int k = java.nio.ByteBuffer.wrap(i).getInt();
            if (!values.containsKey(k))
                values.put(k, 1);
            else {
                long aux = values.get(k);
                aux++;
                values.put(k, aux);
            }
            totalF++;
        }
        fb.close();
        in.close();

        // primeira modificacao
        //for (long v : values.values())
        //    totalF += v;

        IntSet sorted = new IntAVLTreeSet(values.keySet());
        long acumula = 0;
        int b = 0;
        List<String> result = new LinkedList<>();
        long blinha = 0; 
        int last = 0;
        for (int ac : sorted) {
            blinha = values.get(ac);
            acumula += blinha;
            if (acumula > (totalF / (numJCLThreads))) {
                b = ac;
                result.add(b + ":" + acumula);
                acumula = 0;
            }
            last = ac;
        }

        // segunda modificacao
        if(acumula != 0) result.add(last + ":" + acumula);

        JCL_facade jcl = JCL_FacadeImpl.getInstanceLambari();
        jcl.instantiateGlobalVar(id, values);
        sorted.clear();
        sorted = null;
        return result;

    } catch (Exception e) {
        e.printStackTrace();
        return null;
    }
}

public void phase2(int id, int numJCLThreads, String schema) {
    JCL_facade jcl = JCL_FacadeImpl.getInstanceLambari();
    Int2LongMap sorted = (Int2LongMap) jcl.getValue(id).getCorrectResult();
    jcl.deleteGlobalVar(id);
    String[] chunks = schema.split(":");
    int i = 0;
    Int2LongMap[] finais = new Int2LongMap[numJCLThreads];
    for (int r = 0; r < numJCLThreads; r++)
        finais[r] = new Int2LongOpenHashMap();
    for (int ii : sorted.keySet()) {
        i = 0;
        if (ii >= Integer.parseInt(chunks[chunks.length - 1]))
            finais[numJCLThreads - 1].put(ii, sorted.get(ii));
        else {
            if (ii < Integer.parseInt(chunks[0]))
                finais[0].put(ii, sorted.get(ii));
            else {
                tag: {
                    for (int k = 1; k < chunks.length; k++) {
                        if (Integer.parseInt(chunks[i]) <= ii && ii < Integer.parseInt(chunks[k])) {
                            finais[i + 1].put(ii, sorted.get(ii));
                            break tag;
                        }
                        i++;
                    }
                }
            }
        }
    }

    for (int r = 0; r < numJCLThreads; r++) {
        if(!finais[r].isEmpty()) {
            JCL_map<Integer, Int2LongMap> h = new JCLHashMap<>(String.valueOf(r));
            h.put(id, finais[r]);
            long fT = 0;
            for (long kk : finais[r].values())
                fT += kk;
            System.err.println(finais[r].size() + ":" + fT + " phase 2 putting in jcl - thread id " + r);
            finais[r].clear();
            finais[r] = null;
        }
    }
    sorted.clear();
    sorted = null;
    chunks = null;
}

public static Int2LongMap int2LongMap() {
        return new Int2LongOpenHashMap();
//      return new Int2LongAVLTreeMap();
    }

/**
 * Create a new vector with default size.
 */
public LongVector() {
  initialize(Int2LongOpenHashMap.DEFAULT_INITIAL_SIZE);
}

/**
 * Initialize the values of the vector. The default value is 0.0
 *
 * @param size the size of the vector
 */
private void initialize(int size) {
  entries = new Int2LongOpenHashMap(size);
  entries.defaultReturnValue(0L);
}