public void phase3(int id) { Map<Integer, Int2LongMap> h = JCL_FacadeImpl.GetHashMap(String.valueOf(id)); Int2LongMap result = new Int2LongOpenHashMap(); for (Int2LongMap m : h.values()) { for (int i : m.keySet()) { if (result.containsKey(i)) { long j = m.get(i) + result.get(i); result.put(i, j); } else { result.put(i, m.get(i)); } } m.clear(); } long freqT = 0; for (Long v : result.values()) freqT += v; System.err.println("ID: " + id + " size: " + result.size() + " freqT: " + freqT); h.clear(); h.put(id, result); // result.clear(); // result = null; }
/** * Find the parent entity for each. * * @param parents Parent counter map * @param parent Parent storage (output) * @param cnt Count storage (output) */ private void findParent(Int2LongOpenHashMap[] parents, int[] parent, long[] cnt) { parent[0] = 0; cnt[0] = Long.MAX_VALUE; for(int i = 1; i < parents.length; i++) { int best = i; long total = 0, bcount = -1; for(Int2LongMap.Entry p : parents[i].int2LongEntrySet()) { final long c = p.getLongValue(); total += c; if(c > bcount || (c == bcount && c < best)) { bcount = c; best = p.getIntKey(); } } cnt[i] = total; parent[i] = best; } }
public FFMStringFeatureMapModel(@Nonnull FFMHyperParameters params) { super(params); this._w0 = 0.f; this._map = new Int2LongOpenHashMap(DEFAULT_MAPSIZE); _map.defaultReturnValue(-1L); this._buf = new HeapBuffer(HeapBuffer.DEFAULT_CHUNK_SIZE); this._freelistW = new LongArrayList(); this._freelistV = new LongArrayList(); this._initV = true; this._removedV = new RoaringBitmap(); this._numFields = params.numFields; this._entrySizeW = entrySize(1, _useFTRL, _useAdaGrad); this._entrySizeV = entrySize(_factor, _useFTRL, _useAdaGrad); }
private String buildInputDataPartitionSchema(List<JCL_result> r, int numOfJCLThreads){ IntSet sorted = new IntAVLTreeSet(); long totalF=0; Int2LongMap map = new Int2LongOpenHashMap(); for(JCL_result oneR:r){ try{ @SuppressWarnings("unchecked") List<String> l = (List<String>) oneR.getCorrectResult(); for(String s : l){ String[] args = s.split(":"); int key = Integer.parseInt(args[0]); long freq = Long.parseLong(args[1]); sorted.add(key); if(map.containsKey(key)){ freq+=map.get(key); totalF+=map.get(key); } else totalF+=freq; map.put(key, freq); } }catch(Exception e){} } long load=0; int b; String result = ""; for(int ac:sorted){ load += map.get(ac); if(load > (totalF/(numOfJCLThreads))){ b=ac; result += b + ":"; load=0; } } return result; }
/** * Flatten another layers onto a slice. * * @param slice Output slice * @param layer Input layer * @param parents Parents map * @param meta Reduce metadata * @param viewport Viewport */ private void flatten(int[][] slice, int[][] layer, Int2LongOpenHashMap[] parents, Viewport viewport) { // Count the most frequent parent for each entity. for(int y = 0; y < viewport.height; y++) { final int[] rowy = layer[y], outy = slice[y]; for(int x = 0; x < viewport.width; x++) { int id = rowy[x] & 0x00FF_FFFF; // top byte is alpha! if(id == 0) { continue; } parents[id].addTo(outy[x], 1); outy[x] = id; } } }
void createSplitsGreedy(Map<String, Set<OneBlockInfo>> nodeToBlocks, Map<OneBlockInfo, String[]> blockToNodes, Map<String, List<OneBlockInfo>> rackToBlocks, long totLength, int num, long minSizeNode, long minSizeRack, List<InputSplit> splits) { List<OneBlockInfo> blocks = new ArrayList<>(blockToNodes.keySet()); Collections.sort(blocks, new Comparator<OneBlockInfo>() { @Override public int compare(OneBlockInfo o1, OneBlockInfo o2) { return -(int) (o1.length - o2.length); } }); Int2LongOpenHashMap loads = new Int2LongOpenHashMap(); Map<Integer, ArrayList<OneBlockInfo>> parts = new HashMap<>(); Map<Integer, Set<String>> locations = new HashMap<>(); // long num = totLength / maxSize; for (int i = 0; i < num; i ++) { parts.put(i, new ArrayList<OneBlockInfo>()); locations.put(i, new HashSet<String>()); } for (OneBlockInfo blockInfo : blocks) { long min = Long.MAX_VALUE; int selectPart = -1; for (int s = 0; s < num; s ++) { if (loads.get(s) < min) { min = loads.get(s); selectPart = s; } } loads.addTo(selectPart, blockInfo.length); parts.get(selectPart).add(blockInfo); for (String host : blockInfo.hosts) locations.get(selectPart).add(host); } for (Map.Entry<Integer, ArrayList<OneBlockInfo>> entry: parts.entrySet()) { addCreatedSplit(splits, locations.get(entry.getKey()), entry.getValue()); } }
public List<String> phase1(int id, String name, int numJCLThreads) { Int2LongMap values = new Int2LongOpenHashMap(1000000); long totalF = 0; System.err.println("file: " + name); try { File f = new File("../" + name + "/" + name + ".bin"); InputStream in = new BufferedInputStream(new FileInputStream(f)); FastBufferedInputStream fb = new FastBufferedInputStream(in); byte[] i = new byte[4]; while (fb.read(i) == 4) { int k = java.nio.ByteBuffer.wrap(i).getInt(); if (!values.containsKey(k)) values.put(k, 1); else { long aux = values.get(k); aux++; values.put(k, aux); } totalF++; } fb.close(); in.close(); // primeira modificacao //for (long v : values.values()) // totalF += v; IntSet sorted = new IntAVLTreeSet(values.keySet()); long acumula = 0; int b = 0; List<String> result = new LinkedList<>(); long blinha = 0; int last = 0; for (int ac : sorted) { blinha = values.get(ac); acumula += blinha; if (acumula > (totalF / (numJCLThreads))) { b = ac; result.add(b + ":" + acumula); acumula = 0; } last = ac; } // segunda modificacao if(acumula != 0) result.add(last + ":" + acumula); JCL_facade jcl = JCL_FacadeImpl.getInstanceLambari(); jcl.instantiateGlobalVar(id, values); sorted.clear(); sorted = null; return result; } catch (Exception e) { e.printStackTrace(); return null; } }
public void phase2(int id, int numJCLThreads, String schema) { JCL_facade jcl = JCL_FacadeImpl.getInstanceLambari(); Int2LongMap sorted = (Int2LongMap) jcl.getValue(id).getCorrectResult(); jcl.deleteGlobalVar(id); String[] chunks = schema.split(":"); int i = 0; Int2LongMap[] finais = new Int2LongMap[numJCLThreads]; for (int r = 0; r < numJCLThreads; r++) finais[r] = new Int2LongOpenHashMap(); for (int ii : sorted.keySet()) { i = 0; if (ii >= Integer.parseInt(chunks[chunks.length - 1])) finais[numJCLThreads - 1].put(ii, sorted.get(ii)); else { if (ii < Integer.parseInt(chunks[0])) finais[0].put(ii, sorted.get(ii)); else { tag: { for (int k = 1; k < chunks.length; k++) { if (Integer.parseInt(chunks[i]) <= ii && ii < Integer.parseInt(chunks[k])) { finais[i + 1].put(ii, sorted.get(ii)); break tag; } i++; } } } } } for (int r = 0; r < numJCLThreads; r++) { if(!finais[r].isEmpty()) { JCL_map<Integer, Int2LongMap> h = new JCLHashMap<>(String.valueOf(r)); h.put(id, finais[r]); long fT = 0; for (long kk : finais[r].values()) fT += kk; System.err.println(finais[r].size() + ":" + fT + " phase 2 putting in jcl - thread id " + r); finais[r].clear(); finais[r] = null; } } sorted.clear(); sorted = null; chunks = null; }
public static Int2LongMap int2LongMap() { return new Int2LongOpenHashMap(); // return new Int2LongAVLTreeMap(); }
/** * Create a new vector with default size. */ public LongVector() { initialize(Int2LongOpenHashMap.DEFAULT_INITIAL_SIZE); }
/** * Initialize the values of the vector. The default value is 0.0 * * @param size the size of the vector */ private void initialize(int size) { entries = new Int2LongOpenHashMap(size); entries.defaultReturnValue(0L); }