public TrackerDistributedCacheManager(Configuration conf, TaskController controller) throws IOException { this.localFs = FileSystem.getLocal(conf); this.trackerConf = conf; this.lDirAllocator = new LocalDirAllocator("mapred.local.dir"); // setting the cache size to a default of 10GB this.allowedCacheSize = conf.getLong ("local.cache.size", DEFAULT_CACHE_SIZE); // setting the cache number of subdirectories limit to a default of 10000 this.allowedCacheSubdirs = conf.getLong ("mapreduce.tasktracker.cache.local.numberdirectories", DEFAULT_CACHE_SUBDIR_LIMIT); double cleanupPct = conf.getFloat("mapreduce.tasktracker.cache.local.keep.pct", DEFAULT_CACHE_KEEP_AROUND_PCT); this.allowedCacheSizeCleanupGoal = (long)(this.allowedCacheSize * cleanupPct); this.allowedCacheSubdirsCleanupGoal = (long)(this.allowedCacheSubdirs * cleanupPct); this.taskController = controller; this.cleanupThread = new CleanupThread(conf); }
public TrackerDistributedCacheManager(Configuration conf, TaskController controller ) throws IOException { this.localFs = FileSystem.getLocal(conf); this.trackerConf = conf; this.lDirAllocator = new LocalDirAllocator("mapred.local.dir"); // setting the cache size to a default of 10GB this.allowedCacheSize = conf.getLong ("local.cache.size", DEFAULT_CACHE_SIZE); // setting the cache number of subdirectories limit to a default of 10000 this.allowedCacheSubdirs = conf.getLong ("mapreduce.tasktracker.local.cache.numberdirectories", DEFAULT_CACHE_SUBDIR_LIMIT); double cleanupPct = conf.getFloat("mapreduce.tasktracker.cache.local.keep.pct", DEFAULT_CACHE_KEEP_AROUND_PCT); this.allowedCacheSizeCleanupGoal = (long)(this.allowedCacheSize * cleanupPct); this.allowedCacheSubdirsCleanupGoal = (long)(this.allowedCacheSubdirs * cleanupPct); this.taskController = controller; this.cleanupThread = new CleanupThread(conf); }
private void deletePathsInSecureCluster(String newPathName, FileStatus status) throws FileNotFoundException, IOException { // In a secure tasktracker, the subdirectories belong // to different user PathDeletionContext item = null; // iterate and queue subdirectories for cleanup for (FileStatus subDirStatus : localFileSystem.listStatus(status.getPath())) { String owner = subDirStatus.getOwner(); String path = subDirStatus.getPath().getName(); if (path.equals(owner)) { // add it to the cleanup queue item = new TaskController.DeletionContext(taskController, false, owner, newPathName + Path.SEPARATOR_CHAR + path); cleanupQueue.addToQueue(item); } } // queue the parent directory for cleanup item = new TaskController.DeletionContext(taskController, false, status.getOwner(), newPathName); cleanupQueue.addToQueue(item); }
public TrackerDistributedCacheManager(Configuration conf, TaskController taskController) throws IOException { this.localFs = FileSystem.getLocal(conf); this.trackerConf = conf; this.lDirAllocator = new LocalDirAllocator(TTConfig.LOCAL_DIR); this.taskController = taskController; // setting the cache size to a default of 10GB this.allowedCacheSize = conf.getLong(TTConfig.TT_LOCAL_CACHE_SIZE, DEFAULT_CACHE_SIZE); // setting the cache number of subdirectories limit to a default of 10000 this.allowedCacheSubdirs = conf.getLong( TTConfig.TT_LOCAL_CACHE_SUBDIRS_LIMIT, DEFAULT_CACHE_SUBDIR_LIMIT); double cleanupPct = conf.getFloat(TTConfig.TT_LOCAL_CACHE_KEEP_AROUND_PCT, DEFAULT_CACHE_KEEP_AROUND_PCT); this.allowedCacheSizeCleanupGoal = (long)(this.allowedCacheSize * cleanupPct); this.allowedCacheSubdirsCleanupGoal = (long)(this.allowedCacheSubdirs * cleanupPct); this.cleanupThread = new CleanupThread(conf); }
/** * Creates a TrackerDistributedCacheManager with a MRAsyncDiskService. * @param asyncDiskService Provides a set of ThreadPools for async disk * operations. */ public TrackerDistributedCacheManager(Configuration conf, TaskController taskController, MRAsyncDiskService asyncDiskService) throws IOException { this(conf, taskController); this.asyncDiskService = asyncDiskService; }
/** * Create the user log manager to manage user logs on {@link TaskTracker}. * This constructor is there mainly for unit tests. * * @param conf The {@link Configuration} * * @throws IOException */ public UserLogManager(Configuration conf) throws IOException { Class<? extends TaskController> taskControllerClass = conf.getClass("mapred.task.tracker.task-controller", DefaultTaskController.class, TaskController.class); TaskController taskController = (TaskController) ReflectionUtils.newInstance(taskControllerClass, conf); this.taskController = taskController; setFields(conf); }
public MyTrackerDistributedCacheManager(Configuration conf, TaskController controller) throws IOException { super(conf, controller); this.baseDirManager = new TrackerDistributedCacheManager.BaseDirManager() { @Override void checkAndCleanup() throws IOException { throw new RuntimeException("This is a test!!!!"); } }; this.cleanupThread = new TestCleanupThread(conf); }
public MyTrackerDistributedCacheManager(Configuration conf, TaskController controller) throws IOException { super(conf, controller); this.baseDirManager = new TrackerDistributedCacheManager.BaseDirManager() { @Override public void checkAndCleanup() throws IOException { throw new RuntimeException("This is a test!!!!"); } }; this.cleanupThread = new TestCleanupThread(conf); }
@Override protected void setUp() throws IOException,InterruptedException { // Prepare the tests' root dir File TEST_ROOT = new File(TEST_ROOT_DIR); if (!TEST_ROOT.exists()) { TEST_ROOT.mkdirs(); } conf = new Configuration(); conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///"); fs = FileSystem.get(conf); // This test suite will fail if any ancestor directory of the // test directory is not world-searchable (ie +x). // We prefer to fail the test in an obvious manner up front // during setUp() rather than in a subtle way later. assertTrue("Test root directory " + TEST_ROOT + " and all of its " + "parent directories must have a+x permissions", TrackerDistributedCacheManager.ancestorsHaveExecutePermissions( fs, new Path(TEST_ROOT.toString()), new HashMap<URI, FileStatus>())); // Prepare the tests' mapred-local-dir ROOT_MAPRED_LOCAL_DIR = new File(TEST_ROOT_DIR, "mapred/local"); ROOT_MAPRED_LOCAL_DIR.mkdirs(); String []localDirs = new String[numLocalDirs]; for (int i = 0; i < numLocalDirs; i++) { File localDir = new File(ROOT_MAPRED_LOCAL_DIR, "0_" + i); localDirs[i] = localDir.getPath(); localDir.mkdir(); } conf.setStrings("mapred.local.dir", localDirs); Class<? extends TaskController> taskControllerClass = conf.getClass( "mapred.task.tracker.task-controller", DefaultTaskController.class, TaskController.class); taskController = (TaskController) ReflectionUtils.newInstance( taskControllerClass, conf); // setup permissions for mapred local dir UtilsForTests.setupTC(taskController, localDirAllocator, conf.getStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY)); // Create the temporary cache files to be used in the tests. firstCacheFile = new Path(TEST_ROOT_DIR, "firstcachefile"); secondCacheFile = new Path(TEST_ROOT_DIR, "secondcachefile"); firstCacheFilePublic = new Path(TEST_ROOT_DIR, "firstcachefileOne"); secondCacheFilePublic = new Path(TEST_ROOT_DIR, "secondcachefileOne"); createPublicTempFile(firstCacheFilePublic); createPublicTempFile(secondCacheFilePublic); createPrivateTempFile(firstCacheFile); createPrivateTempFile(secondCacheFile); firstCacheDirPublic = new Path(TEST_ROOT_DIR, "firstcachedirPublic"); firstCacheDirPrivate = new Path(TEST_ROOT_DIR, "firstcachedirPrivate"); firstCacheFileInDirPublic = new Path(firstCacheDirPublic, "firstcacheFileinDirPublic.txt"); firstCacheFileInDirPrivate = new Path(firstCacheDirPrivate, "firstcacheFileinDirPrivate.txt"); createPublicTempDir(firstCacheDirPublic); createPrivateTempDir(firstCacheDirPrivate); createPublicTempFile(firstCacheFileInDirPublic); createPrivateTempFile(firstCacheFileInDirPrivate); }
@Override protected void setUp() throws IOException,InterruptedException { // Prepare the tests' root dir File TEST_ROOT = new File(TEST_ROOT_DIR); if (!TEST_ROOT.exists()) { TEST_ROOT.mkdirs(); } // Prepare the tests' mapred-local-dir ROOT_MAPRED_LOCAL_DIR = new File(TEST_ROOT_DIR, "mapred/local"); ROOT_MAPRED_LOCAL_DIR.mkdirs(); String []localDirs = new String[numLocalDirs]; for (int i = 0; i < numLocalDirs; i++) { File localDir = new File(ROOT_MAPRED_LOCAL_DIR, "0_" + i); localDirs[i] = localDir.getPath(); localDir.mkdir(); } conf = new Configuration(); conf.setStrings("mapred.local.dir", localDirs); conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///"); fs = FileSystem.get(conf); Class<? extends TaskController> taskControllerClass = conf.getClass( "mapred.task.tracker.task-controller", DefaultTaskController.class, TaskController.class); taskController = (TaskController) ReflectionUtils.newInstance( taskControllerClass, conf); // setup permissions for mapred local dir UtilsForTests.setupTC(taskController, localDirAllocator, conf.getStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY)); // Create the temporary cache files to be used in the tests. firstCacheFile = new Path(TEST_ROOT_DIR, "firstcachefile"); secondCacheFile = new Path(TEST_ROOT_DIR, "secondcachefile"); firstCacheFilePublic = new Path(TEST_ROOT_DIR, "firstcachefileOne"); secondCacheFilePublic = new Path(TEST_ROOT_DIR, "secondcachefileOne"); createPublicTempFile(firstCacheFilePublic); createPublicTempFile(secondCacheFilePublic); createPrivateTempFile(firstCacheFile); createPrivateTempFile(secondCacheFile); firstCacheDirPublic = new Path(TEST_ROOT_DIR, "firstcachedirPublic"); firstCacheDirPrivate = new Path(TEST_ROOT_DIR, "firstcachedirPrivate"); firstCacheFileInDirPublic = new Path(firstCacheDirPublic, "firstcacheFileinDirPublic.txt"); firstCacheFileInDirPrivate = new Path(firstCacheDirPrivate, "firstcacheFileinDirPrivate.txt"); createPublicTempDir(firstCacheDirPublic); createPrivateTempDir(firstCacheDirPrivate); createPublicTempFile(firstCacheFileInDirPublic); createPrivateTempFile(firstCacheFileInDirPrivate); }
@Override protected void setUp() throws IOException,InterruptedException { // Prepare the tests' root dir File TEST_ROOT = new File(TEST_ROOT_DIR); if (!TEST_ROOT.exists()) { TEST_ROOT.mkdirs(); } conf = new Configuration(); conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///"); fs = FileSystem.get(conf); // This test suite will fail if any ancestor directory of the // test directory is not world-searchable (ie +x). // We prefer to fail the test in an obvious manner up front // during setUp() rather than in a subtle way later. assertTrue("Test root directory " + TEST_ROOT + " and all of its " + "parent directories must have a+x permissions", ClientDistributedCacheManager.ancestorsHaveExecutePermissions( fs, new Path(TEST_ROOT.toString()))); // Prepare the tests' mapred-local-dir ROOT_MAPRED_LOCAL_DIR = new File(TEST_ROOT_DIR, "mapred/local"); ROOT_MAPRED_LOCAL_DIR.mkdirs(); String []localDirs = new String[numLocalDirs]; for (int i = 0; i < numLocalDirs; i++) { File localDir = new File(ROOT_MAPRED_LOCAL_DIR, "0_" + i); localDirs[i] = localDir.getPath(); localDir.mkdir(); } conf.setStrings(MRConfig.LOCAL_DIR, localDirs); Class<? extends TaskController> taskControllerClass = conf.getClass( TTConfig.TT_TASK_CONTROLLER, DefaultTaskController.class, TaskController.class); taskController = (TaskController) ReflectionUtils.newInstance( taskControllerClass, conf); // setup permissions for mapred local dir taskController.setup(); // Create the temporary cache files to be used in the tests. firstCacheFile = new Path(TEST_ROOT_DIR, "firstcachefile"); secondCacheFile = new Path(TEST_ROOT_DIR, "secondcachefile"); createPrivateTempFile(firstCacheFile); createPrivateTempFile(secondCacheFile); }
/** * Create the user log manager to manage user logs on {@link TaskTracker}. * * It should be explicitly started using {@link #start()} to start functioning * * @param conf The {@link Configuration} * @param taskController The task controller to delete the log files * * @throws IOException */ public UserLogManager(Configuration conf, TaskController taskController) throws IOException { this.taskController = taskController; setFields(conf); }
/** * Get the taskController for deleting logs. * @return the TaskController */ public TaskController getTaskController() { return taskController; }
/** * Create a Localizer instance * * @param fileSys * @param lDirs * @param tc */ public Localizer(FileSystem fileSys, String[] lDirs, TaskController tc) { fs = fileSys; localDirs = lDirs; taskController = tc; }
