Java 类org.apache.commons.io.ByteOrderMark 实例源码

项目：Moenagade 文件：BOMInputStream.java

/**
 * Return the BOM (Byte Order Mark).
 *
 * @return The BOM or null if none
 * @throws IOException if an error reading the first bytes of the stream occurs
 */
public ByteOrderMark getBOM() throws IOException {
    if (firstBytes == null) {
        int max = 0;
        for (ByteOrderMark bom : boms) {
            max = Math.max(max, bom.length());
        }
        firstBytes = new int[max];
        for (int i = 0; i < firstBytes.length; i++) {
            firstBytes[i] = in.read();
            fbLength++;
            if (firstBytes[i] < 0) {
                break;
            }

            byteOrderMark = find();
            if (byteOrderMark != null) {
                if (!include) {
                    fbLength = 0;
                }
                break;
            }
        }
    }
    return byteOrderMark;
}

项目：dremio-oss 文件：TextRecordWriter.java

@Override
public void startPartition(WritePartition partition) throws Exception {

  if(this.partition != null){
    close();
  }

  this.partition = partition;
  // open a new file for writing data with new schema
  try {
    this.path = fs.canonicalizePath(partition.qualified(location, prefix + "_" + index + "." + extension));
    DataOutputStream fos = fs.create(path);
    stream = new PrintStream(fos);
    stream.write(ByteOrderMark.UTF_8.getBytes(), 0, ByteOrderMark.UTF_8.length());
    logger.debug("Created file: {}", path);
  } catch (IOException e) {
    throw UserException.dataWriteError(e)
      .message("Failure while attempting to write file %s.", path)
      .build(logger);
  }
  index++;

  stream.print(Joiner.on(fieldDelimiter).join(columnNames));
  stream.print(lineDelimiter);

}

项目：dremio-oss 文件：TextInput.java

private final boolean checkBom(ByteOrderMark bom) {
  int bomLength = bom.length();
  if (bufferPtr + bomLength >= length) {
    // Not enough bytes from the current position to the end of the buffer
    return false;
  }
  if (BoundsChecking.BOUNDS_CHECKING_ENABLED) {
    buffer.checkBytes(bufferPtr - 1, bufferPtr + bomLength);
  }

  byte[] bomBytes = bom.getBytes();
  for (int i = 0; i < bomLength; i++) {
    byte nextChar = PlatformDependent.getByte(bStartMinus1 + bufferPtr + i);
    if (nextChar != bomBytes[i]) {
      // No BOM. Position is unchanged
      return false;
    }
  }
  return true;
}

项目：dremio-oss 文件：TestNewTextReader.java

@Test
public void testBomUtf8() throws Exception {
  // Simple .csv file with a UTF-8 BOM. Should read successfully
  File testFolder = tempDir.newFolder("testUtf8Folder");
  File testFile = new File(testFolder, "utf8.csv");
  PrintStream p = new PrintStream(testFile);
  p.write(ByteOrderMark.UTF_8.getBytes(), 0, ByteOrderMark.UTF_8.length());
  p.print("A,B\n");
  p.print("5,7\n");
  p.close();

  testBuilder()
    .sqlQuery(String.format("select * from table(dfs.`%s` (type => 'text', " +
      "fieldDelimiter => ',', lineDelimiter => '\n', extractHeader => true))",
      testFile.getAbsolutePath()))
    .unOrdered()
    .baselineColumns("A","B")
    .baselineValues("5", "7")
    .go();
}

项目：dremio-oss 文件：TestNewTextReader.java

@Test
public void testErrorBomUtf16() throws Exception {
  // UTF-16 BOM should cause a dataReadError user exception
  File testFolder = tempDir.newFolder("testUtf16Folder");
  File testFile = new File(testFolder, "utf16.csv");
  PrintStream p = new PrintStream(testFile);
  p.write(ByteOrderMark.UTF_16LE.getBytes(), 0, ByteOrderMark.UTF_16LE.length());
  p.print("A,B\n");
  p.print("5,7\n");
  p.close();

  thrownException.expect(new UserExceptionMatcher(UserBitShared.DremioPBError.ErrorType.DATA_READ,
    "DATA_READ ERROR: UTF-16 files not supported"));
  // NB: using test() instead of testBuilder() because it unwraps the thrown RpcException and re-throws the
  // underlying UserException (which is then matched with the UserExceptionMatcher)
  test(String.format("select * from table(dfs.`%s` (type => 'text', " +
      "fieldDelimiter => ',', lineDelimiter => '\n', extractHeader => true))",
    testFile.getAbsolutePath()));
}