public static ResourceLocalizationSpec newResourceLocalizationSpec( LocalResource rsrc, Path path) { URL local = ConverterUtils.getYarnUrlFromPath(path); ResourceLocalizationSpec resourceLocalizationSpec = Records.newRecord(ResourceLocalizationSpec.class); resourceLocalizationSpec.setDestinationDirectory(local); resourceLocalizationSpec.setResource(rsrc); return resourceLocalizationSpec; }
@Override public synchronized URL getResource() { LocalResourceProtoOrBuilder p = viaProto ? proto : builder; if (this.url != null) { return this.url; } if (!p.hasResource()) { return null; } this.url = convertFromProtoFormat(p.getResource()); return this.url; }
/** * return a hadoop path from a given url * * @param url * url to convert * @return path from {@link URL} * @throws URISyntaxException */ public static Path getPathFromYarnURL(URL url) throws URISyntaxException { String scheme = url.getScheme() == null ? "" : url.getScheme(); String authority = ""; if (url.getHost() != null) { authority = url.getHost(); if (url.getUserInfo() != null) { authority = url.getUserInfo() + "@" + authority; } if (url.getPort() > 0) { authority += ":" + url.getPort(); } } return new Path( (new URI(scheme, authority, url.getFile(), null, null)).normalize()); }
static ResourceLocalizationSpec getMockRsrc(Random r, LocalResourceVisibility vis, Path p) { ResourceLocalizationSpec resourceLocalizationSpec = mock(ResourceLocalizationSpec.class); LocalResource rsrc = mock(LocalResource.class); String name = Long.toHexString(r.nextLong()); URL uri = mock(org.apache.hadoop.yarn.api.records.URL.class); when(uri.getScheme()).thenReturn("file"); when(uri.getHost()).thenReturn(null); when(uri.getFile()).thenReturn("/local/" + vis + "/" + name); when(rsrc.getResource()).thenReturn(uri); when(rsrc.getSize()).thenReturn(r.nextInt(1024) + 1024L); when(rsrc.getTimestamp()).thenReturn(r.nextInt(1024) + 2048L); when(rsrc.getType()).thenReturn(LocalResourceType.FILE); when(rsrc.getVisibility()).thenReturn(vis); when(resourceLocalizationSpec.getResource()).thenReturn(rsrc); when(resourceLocalizationSpec.getDestinationDirectory()). thenReturn(ConverterUtils.getYarnUrlFromPath(p)); return resourceLocalizationSpec; }
static LocalResourceStatus createLocalResourceStatus() { LocalResourceStatus ret = recordFactory.newRecordInstance(LocalResourceStatus.class); assertTrue(ret instanceof LocalResourceStatusPBImpl); ret.setResource(createResource()); ret.setLocalPath( URL.fromPath( new Path("file:///local/foo/bar"))); ret.setStatus(ResourceStatusType.FETCH_SUCCESS); ret.setLocalSize(4443L); Exception e = new Exception("Dingos."); e.setStackTrace(new StackTraceElement[] { new StackTraceElement("foo", "bar", "baz", 10), new StackTraceElement("sbb", "one", "onm", 10) }); ret.setException(SerializedException.newInstance(e)); return ret; }
static LocalizerHeartbeatResponse createLocalizerHeartbeatResponse() throws URISyntaxException { LocalizerHeartbeatResponse ret = recordFactory.newRecordInstance(LocalizerHeartbeatResponse.class); assertTrue(ret instanceof LocalizerHeartbeatResponsePBImpl); ret.setLocalizerAction(LocalizerAction.LIVE); LocalResource rsrc = createResource(); ArrayList<ResourceLocalizationSpec> rsrcs = new ArrayList<ResourceLocalizationSpec>(); ResourceLocalizationSpec resource = recordFactory.newRecordInstance(ResourceLocalizationSpec.class); resource.setResource(rsrc); resource.setDestinationDirectory( URL.fromPath((new Path("/tmp" + System.currentTimeMillis())))); rsrcs.add(resource); ret.setResourceSpecs(rsrcs); System.out.println(resource); return ret; }
private LocalResource createLocalResource(Path resourcePath, LocalResourceType resourceType, LocalResourceVisibility resourceVisibility) { LocalResource localResource = Records.newRecord(LocalResource.class); URL resourceUrl = ConverterUtils.getYarnUrlFromPath(resourcePath); try { FileStatus resourceFileStatus = resourcePath.getFileSystem(yarnConfiguration).getFileStatus(resourcePath); if (null == resourceFileStatus) { throw new LocalizerResourceException("Check getFileStatus implementation. getFileStatus gets unexpected null for resourcePath " + resourcePath); } localResource.setResource(resourceUrl); log.info("setLocalizerResource for {}", resourceUrl); localResource.setSize(resourceFileStatus.getLen()); localResource.setTimestamp(resourceFileStatus.getModificationTime()); localResource.setType(resourceType); localResource.setVisibility(resourceVisibility); return localResource; } catch (IOException ioe) { log.error("IO Exception when accessing the resource file status from the filesystem: " + resourcePath, ioe); throw new LocalizerResourceException("IO Exception when accessing the resource file status from the filesystem: " + resourcePath); } }
static LocalResource createJar(FileContext files, Path p, LocalResourceVisibility vis) throws IOException { LOG.info("Create jar file " + p); File jarFile = new File((files.makeQualified(p)).toUri()); FileOutputStream stream = new FileOutputStream(jarFile); LOG.info("Create jar out stream "); JarOutputStream out = new JarOutputStream(stream, new Manifest()); LOG.info("Done writing jar stream "); out.close(); LocalResource ret = recordFactory.newRecordInstance(LocalResource.class); ret.setResource(URL.fromPath(p)); FileStatus status = files.getFileStatus(p); ret.setSize(status.getLen()); ret.setTimestamp(status.getModificationTime()); ret.setType(LocalResourceType.PATTERN); ret.setVisibility(vis); ret.setPattern("classes/.*"); return ret; }
static LocalResource createJarFile(FileContext files, Path p, int len, Random r, LocalResourceVisibility vis) throws IOException, URISyntaxException { byte[] bytes = new byte[len]; r.nextBytes(bytes); File archiveFile = new File(p.toUri().getPath() + ".jar"); archiveFile.createNewFile(); JarOutputStream out = new JarOutputStream( new FileOutputStream(archiveFile)); out.putNextEntry(new JarEntry(p.getName())); out.write(bytes); out.closeEntry(); out.close(); LocalResource ret = recordFactory.newRecordInstance(LocalResource.class); ret.setResource(URL.fromPath(new Path(p.toString() + ".jar"))); ret.setSize(len); ret.setType(LocalResourceType.ARCHIVE); ret.setVisibility(vis); ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".jar")) .getModificationTime()); return ret; }
static ResourceLocalizationSpec getMockRsrc(Random r, LocalResourceVisibility vis, Path p) { ResourceLocalizationSpec resourceLocalizationSpec = mock(ResourceLocalizationSpec.class); LocalResource rsrc = mock(LocalResource.class); String name = Long.toHexString(r.nextLong()); URL uri = mock(org.apache.hadoop.yarn.api.records.URL.class); when(uri.getScheme()).thenReturn("file"); when(uri.getHost()).thenReturn(null); when(uri.getFile()).thenReturn("/local/" + vis + "/" + name); when(rsrc.getResource()).thenReturn(uri); when(rsrc.getSize()).thenReturn(r.nextInt(1024) + 1024L); when(rsrc.getTimestamp()).thenReturn(r.nextInt(1024) + 2048L); when(rsrc.getType()).thenReturn(LocalResourceType.FILE); when(rsrc.getVisibility()).thenReturn(vis); when(resourceLocalizationSpec.getResource()).thenReturn(rsrc); when(resourceLocalizationSpec.getDestinationDirectory()). thenReturn(URL.fromPath(p)); return resourceLocalizationSpec; }
private static void addToLocalResources(FileSystem fs, String key, Path dst, Map<String, LocalResource> localResources) throws IOException { FileStatus scFileStatus = fs.getFileStatus(dst); LocalResource resource = LocalResource.newInstance( URL.fromURI(dst.toUri()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime()); localResources.put(key, resource); }
@Override public synchronized void setResource(URL resource) { maybeInitBuilder(); if (resource == null) builder.clearResource(); this.url = resource; }
public static URL getYarnUrlFromURI(URI uri) { URL url = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(URL.class); if (uri.getHost() != null) { url.setHost(uri.getHost()); } if (uri.getUserInfo() != null) { url.setUserInfo(uri.getUserInfo()); } url.setPort(uri.getPort()); url.setScheme(uri.getScheme()); url.setFile(uri.getPath()); return url; }
@Test public void testConvertUrlWithNoPort() throws URISyntaxException { Path expectedPath = new Path("hdfs://foo.com"); URL url = ConverterUtils.getYarnUrlFromPath(expectedPath); Path actualPath = ConverterUtils.getPathFromYarnURL(url); assertEquals(expectedPath, actualPath); }
public static LocalResource newLocalResource(URL url, LocalResourceType type, LocalResourceVisibility visibility, long size, long timestamp, boolean shouldBeUploadedToSharedCache) { LocalResource resource = recordFactory.newRecordInstance(LocalResource.class); resource.setResource(url); resource.setType(type); resource.setVisibility(visibility); resource.setSize(size); resource.setTimestamp(timestamp); resource.setShouldBeUploadedToSharedCache(shouldBeUploadedToSharedCache); return resource; }
public static URL newURL(String scheme, String host, int port, String file) { URL url = recordFactory.newRecordInstance(URL.class); url.setScheme(scheme); url.setHost(host); url.setPort(port); url.setFile(file); return url; }
@Override public URL getLocalPath() { LocalResourceStatusProtoOrBuilder p = viaProto ? proto : builder; if (this.localPath != null) { return this.localPath; } if (!p.hasLocalPath()) { return null; } this.localPath = convertFromProtoFormat(p.getLocalPath()); return this.localPath; }
@Override public void setLocalPath(URL localPath) { maybeInitBuilder(); if (localPath == null) builder.clearLocalPath(); this.localPath = localPath; }
MockLocalResourceStatus(LocalResource rsrc, ResourceStatusType tag, URL localPath, SerializedException ex) { this.rsrc = rsrc; this.tag = tag; this.localPath = localPath; this.ex = ex; }
private static LocalResource getMockedResource(Random r, LocalResourceVisibility vis) { String name = Long.toHexString(r.nextLong()); URL url = getPath("/local/PRIVATE/" + name); LocalResource rsrc = BuilderUtils.newLocalResource(url, LocalResourceType.FILE, vis, r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L, false); return rsrc; }
private static Entry<String, LocalResource> getMockRsrc(Random r, LocalResourceVisibility vis) { String name = Long.toHexString(r.nextLong()); URL url = BuilderUtils.newURL("file", null, 0, "/local" + vis + "/" + name); LocalResource rsrc = BuilderUtils.newLocalResource(url, LocalResourceType.FILE, vis, r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L, false); return new SimpleEntry<String, LocalResource>(name, rsrc); }
/** * Create a {@link LocalResource} record with all the given parameters. */ private static LocalResource createLocalResource(FileSystem fc, Path file, LocalResourceType type, LocalResourceVisibility visibility) throws IOException { FileStatus fstat = fc.getFileStatus(file); URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat .getPath())); long resourceSize = fstat.getLen(); long resourceModificationTime = fstat.getModificationTime(); return LocalResource.newInstance(resourceURL, type, visibility, resourceSize, resourceModificationTime); }
