How can I decode the UTF-8 codepoints of a MappedByteBuffer in Java without copying the buffer? Copying the buffer into memory would defeat the point of mapping the memory, and since CharsetDecoder would require me to copy the buffer with the toCharArray method, it would defeat the purpose of mapping the memory. Is there any way to efficiently decode it without copying the buffer?
Here is a somewhat more complete answer.
It reads the Input-File in Chunks & stores the result in a CharBuffer.
import static java.nio.charset.StandardCharsets.UTF_8;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CoderResult;
import java.nio.charset.MalformedInputException;
import java.nio.charset.UnmappableCharacterException;
import java.nio.file.Files;
import java.nio.file.Path;
public class ChunkedUtf8Decoder {
/*
* Valid UTF-8 Multi-Bytes
* -----------------------
* 1-Byte : 0xxxxxxx
* 2-Byte : 110xxxxx + 10xxxxxx
* 3-Byte : 1110xxxx + 10xxxxxx + 10xxxxxx
* 4-Byte : 11110xxx + 10xxxxxx + 10xxxxxx + 10xxxxxx
*/
private static final int UTF8_CONTINUE_MASK = 0b11_000000;
private static final int UTF8_INVALID_MASK = 0b11111_000;
private static final int UTF8_CONTINUE_PREFIX = 0b10_000000;
private static final int UTF8_INVALID_PREFIX = UTF8_INVALID_MASK;
private static final int CHUNK_SIZE = 24; // TODO Test value! Try something larger, e.g. 64_000
private final CharsetDecoder utf8Decoder = UTF_8.newDecoder();
private final ByteBuffer bb = ByteBuffer.allocate(CHUNK_SIZE); // TODO maybe allocateDirect?
private final CharBuffer cb;
private final Path inputPath;
private byte[] remainder = {};
private int bytesChunked = 0;
public ChunkedUtf8Decoder(final Path inputPath) throws IOException {
this.inputPath = inputPath;
this.cb = CharBuffer.allocate(Math.toIntExact(Files.size(inputPath)));
// this.utf8Decoder.onMalformedInput (CodingErrorAction.REPLACE); // (TODO Default is REPORT)
// this.utf8Decoder.onUnmappableCharacter(CodingErrorAction.REPLACE); // (TODO Default is REPORT)
}
/**
* Split the Input-File into Chunks & Decode them, appending the result to our CharBuffer.
*
* #throws IOException
*/
public void decode() throws IOException {
try(final FileChannel channel = FileChannel.open(inputPath))
{
while (channel.read(bb) != -1) {
debugBytesIn("Read......:", false);
this.remainder = backupToCharBoundary();
debugBytesIn("Backed up.:", true);
this.bytesChunked += decodeChunk();
bb.clear(); // (position=0, limit=capacity, mark=-1)
bb.put(this.remainder); // (Final Remainder is dealt with below)
}
debugBytesIn("EOF.......:", false);
/*
* Lastly, deal with Final Remainder (0 to 4 Bytes) # start of Buffer...
*/
decodeChunk();
}
}
/**
* We try to back up the BB to a Character boundary.
* If the file is correctly encoded, the shorter BB will then decode OK.
* The Remainder will be processed after the NEXT Read operation.
*
* #return
* #throws MalformedInputException
*/
private byte[] backupToCharBoundary() throws MalformedInputException {
for (int i = 1; i <= 4; i++) {
final int trailingBytePos = bb.position() - i;
if (trailingBytePos < 0) {
/*
* If there were too few Bytes available, carry them over until either
* a) more Bytes become available, or
* b) EOF is reached
*/
final byte[] remainder = new byte[bb.position()];
bb.get (0, remainder);
bb.clear(); // (position=0, limit=capacity, mark=-1)
return remainder; // (Entire contents of BB in Remainder, BB Empty)
}
final int trailingByte = 0xFF & bb.get(trailingBytePos);
/*
* We stop as soon as we encounter a Character-Selector...
* (The following 2 intervals are either invalid or a continuation-character)
*/
if ((trailingByte & UTF8_INVALID_MASK ) != UTF8_INVALID_PREFIX // 11111xxx
&& (trailingByte & UTF8_CONTINUE_MASK) != UTF8_CONTINUE_PREFIX) { // 10xxxxxx
/*
* OK, we have found a (1-, 2-, 3- or 4-Byte) Character-Selector...
*/
final byte[] remainder = new byte[i];
bb.get (trailingBytePos, remainder);
bb.position(trailingBytePos);
return remainder; // (Trailing 1-4 Bytes of BB in Remainder)
}
}
/*
* Although at least 4 Bytes are available, We couldn't find a Character-Selector!
* This is an error.
* We leave the ByteBuffer unchanged & return an empty Remainder.
* The CharsetDecoder will produce a MalformedInputException in due course...
*/
return new byte[0];
}
/**
* Decode the current Chunk of Bytes & append the result to our CB.
*
* #return
* #throws CharacterCodingException
*/
private int decodeChunk() throws CharacterCodingException {
try {
this.bb.flip(); // (limit=position, position=0, mark=-1)
; this.utf8Decoder.reset();
final CoderResult res = this.utf8Decoder.decode(this.bb, this.cb, true);
System.out.println("CB Chars..: " + this.cb.position());
/*
* Why on earth is UNDERFLOW the correct result of an error-free Decode?!
*/
if (res.isUnderflow() == false) {
res.throwException();
}
return this.bb.position();
}
catch (final MalformedInputException e) {
throw new MalformedInputException (this.bytesChunked + e.getInputLength());
}
catch (final UnmappableCharacterException e) {
throw new UnmappableCharacterException(this.bytesChunked + e.getInputLength());
/*
* (Famous Last Words "UnmappableCharacterException never occurs with UTF-8 Decoder")
*/
}
catch (final CharacterCodingException inputUnderflowOrOutputOverflow) {
throw inputUnderflowOrOutputOverflow;
/*
* (Underflow if the final Character of the final Chunk was incomplete)
* (Overflow if the Output Buffer is too small, which SHOULD NOT HAPPEN with our big CB)
*/
}
}
/**
* Some Debug Writes.
*
* #param debugText
* #param includeRemainder
*/
private void debugBytesIn(final String debugText, final boolean includeRemainder) {
System .out.print(debugText + " previous=" + this.bytesChunked + " bbPos=" + bb.position() + " Bytes.:");
for ( int p = 0; p < bb.position(); p++) {
System .out.print(" " + Integer.toHexString(0xFF & bb.get(p)));
}
if (includeRemainder) {
System .out.print(" Remainder.:");
for (int r = 0; r < this.remainder.length; r++) {
System.out.print(" " + Integer.toHexString(0xFF & this.remainder[r]));
}
}
System .out.println();
}
public static void main(final String[] args) throws IOException {
final ChunkedUtf8Decoder utf8Decoder = new ChunkedUtf8Decoder(Path.of("Utf8-Chars_nn.txt"));
; utf8Decoder.decode();
System.out.println (utf8Decoder.cb.flip()); // TODO Note.: flip()
}
}
Not really: essentially a char[] needs to be built from the byte[] (either direct or indirect) which is backing the MappedByteBuffer.
So something like the following is necessary:
private static CharBuffer readUTF8(final Path path) throws IOException {
final long byteCount = path.toFile().length();
final FileChannel channel = FileChannel.open(PATH);
final MappedByteBuffer byteBuffer = channel.map(MapMode.READ_ONLY, 0, byteCount);
return StandardCharsets.UTF_8.decode(byteBuffer);
}
The following snippet demonstrates the possibility to reuse various components used in the above example.
As stated in the Javadoc, this is a complex matter.
So it should only be considered as a collection of ideas & invocations of various JDK methods, which may or may not be of use, depending on your concrete requirements.
You really need an in-depth understanding of Charsets etc. before using the following...
/**
* It is possible to pre-allocate a CharsetDecoder & CharBuffer & use those multiple times.
* Be aware that this would make your logic MUCH more complicated.
* a) you'll have to control use/reuse & initialisation of the CharsetDecoder
* b) you'll need to ensure the CharBuffer has sufficient capacity
*
* The following is just snippets which may be of use.
*/
public static void main(final String[] args) throws Exception {
final CharsetDecoder utf8Decoder = StandardCharsets.UTF_8.newDecoder();
; utf8Decoder.onMalformedInput (CodingErrorAction.REPLACE);
; utf8Decoder.onUnmappableCharacter(CodingErrorAction.REPLACE);
final int charBufLength = 0; // TODO how long?
final CharBuffer charBuf = CharBuffer.allocate(charBufLength);
final int byteBufferLength = 0; // TODO how long?
final MappedByteBuffer byteBuffer = FileChannel.open(Path.of("my File")).map(MapMode.READ_ONLY, 0, byteBufferLength);
readUTF8(utf8Decoder, byteBuffer, charBuf);
}
private static void readUTF8(final CharsetDecoder utf8Decoder, final MappedByteBuffer byteBuffer, final CharBuffer charBuffer) throws IOException {
/* Maybe */ utf8Decoder.reset(); // TODO check out the source of the following decode(byteBuffer) method
/* Maybe */ utf8Decoder.decode(byteBuffer); // TODO check out the source of this method
utf8Decoder.decode(byteBuffer, charBuffer, true); // reuses utf8Decoder & charBuffer
charBuffer.flip();
}
Related
I'm build a blockchain app.
When I run tests in main, no matter what I do, no matter how much time I give it, when I log different things out, I'm unable to get 4 leading zeroes and so complete a difficulty level of 4. I see the log of the binary hashes and many times they have repeating elements, 1111 for instance, but never 0000 until my time is hit and the difficulty decreases to three. I have no idea why.
I borrowed the hash algorithm from an online source and I checked its output against an online hasher and it checked out.
I know with each level of difficulty it increases exponentially but 2^4 is still only 16 and I see other repeating numbers (1111, 1010, any combination except 0000). Is there any reason why this might be the case?
I wanted to provide an abundance of code rather than a shortage. Logically it makes no sense why randomly if all numbers were equally possible, it woudln't turn up 0000* (e.g. 0000101011at some point). therefore Four zeros must not be possible, but why? I waited 100 seconds mutliple times and saw other numbers repeat themselves. I saw it hit at exactly 4 or 3 or 2 seconds each time on the dot when difficulty went to three. When I start at difficulty 5 (genesis block) it will never solve- I'm sure even if I left it running overnight. So what could be going on?
package privblock.gerald.ryan;
import java.nio.charset.StandardCharsets;
import java.security.NoSuchAlgorithmException;
import java.time.Instant;
import java.util.Arrays;
import java.util.Date; // gets time in ms.
import privblock.gerald.ryan.util.CryptoHash;
/**
*
* #author Gerald Ryan Block Class of blockchain app
*
* Description: The block hash is the result of the timestamp, the
* last_hash, the data, the difficulty and the nonce
*
*/
public class Block {
long timestamp;
String lastHash;
String hash;
String[] data;
int difficulty;
int nonce;
// Millisecond basis
;
static long MILLISECONDS = 1;
static long SECONDS = 1000 * MILLISECONDS;
static long MINE_RATE = 2 * SECONDS;
/**
* A block is a unit of storage for a blockchain that supports a cryptocurrency.
*
* #param timestamp
* #param lastHash
* #param hash
* #param data
* #param difficulty
* #param nonce
*/
public Block(long timestamp, String lastHash, String hash, String[] data, int difficulty, int nonce) {
super();
this.timestamp = timestamp;
this.lastHash = lastHash;
this.hash = hash;
this.data = data;
this.difficulty = difficulty;
this.nonce = nonce;
}
public String toString() {
return "\n-----------BLOCK--------\ntimestamp: " + this.timestamp + "\nlastHash: " + this.lastHash + "\nhash: "
+ this.hash + "\ndifficulty: " + this.getDifficulty() + "\nNonce: " + this.nonce
+ "\n-----------------------\n";
}
/**
* Mine a block based on given last block and data until a block hash is found
* that meets the leading 0's Proof of Work requirement.
*
* #param last_block
* #param data
* #return
* #throws NoSuchAlgorithmException
*/
public static Block mine_block(Block last_block, String[] data) throws NoSuchAlgorithmException {
long timestamp = new Date().getTime();
String last_hash = last_block.getHash();
int difficulty = Block.adjust_difficulty(last_block, timestamp);
int nonce = 0;
String hash = CryptoHash.getSHA256(timestamp, last_block.getHash(), data, difficulty, nonce);
String proof_of_work = CryptoHash.n_len_string('0', difficulty);
// System.out.println("Proof of work " + proof_of_work);
String binary_hash = CryptoHash.hex_to_binary(hash);
// System.out.println("binary hash " + binary_hash);
String binary_hash_work_end = binary_hash.substring(0, difficulty);
// System.out.println("binary_Hash_work_end " + binary_hash_work_end);
System.out.println("Difficulty: " + difficulty);
while (!proof_of_work.equalsIgnoreCase(binary_hash_work_end)) {
// System.out.println("Working");
nonce += 1;
timestamp = new Date().getTime();
difficulty = Block.adjust_difficulty(last_block, timestamp);
hash = CryptoHash.getSHA256(timestamp, last_block.getHash(), data, difficulty, nonce);
proof_of_work = CryptoHash.n_len_string('0', difficulty);
binary_hash = CryptoHash.hex_to_binary(hash);
binary_hash_work_end = binary_hash.substring(0, difficulty);
// System.out.println(binary_hash_work_end);
// System.out.println(binary_hash);
// System.out.println(proof_of_work);
}
System.out.println("Solved at Difficulty: " + difficulty);
// System.out.println("Proof of work requirement " + proof_of_work);
// System.out.println("binary_Hash_work_end " + binary_hash_work_end);
// System.out.println("binary hash " + binary_hash);
System.out.println("BLOCK MINED");
return new Block(timestamp, last_hash, hash, data, difficulty, nonce);
}
/**
* Generate Genesis block
*
* #return
*/
public static Block genesis_block() {
long timestamp = 1;
String last_hash = "genesis_last_hash";
String hash = "genesis_hash";
String[] data = { "buy", "privcoin" };
int difficulty = 4;
int nonce = 0;
return new Block(timestamp, last_hash, hash, data, difficulty, nonce);
}
/**
* Calculate the adjusted difficulty according to the MINE_RATE. Increase the
* difficulty for quickly mined blocks. Decrease the difficulty for slowly mined
* blocks.
*
* #param last_block
* #param new_timestamp
*/
public static int adjust_difficulty(Block last_block, long new_timestamp) {
long time_diff = new_timestamp - last_block.getTimestamp();
// System.out.println(time_diff);
if (time_diff < MINE_RATE) {
// System.out.println("Increasing difficulty");
return last_block.getDifficulty() + 1;
} else if (last_block.getDifficulty() - 1 > 0) {
// System.out.println("Decreasing difficulty");
return last_block.getDifficulty() - 1;
} else {
return 1;
}
}
/**
* Validate block by enforcing following rules: - Block must have the proper
* last_hash reference - Block must meet the proof of work requirements -
* difficulty must only adjust by one - block hash must be a valid combination
* of block fields
*
* #param last_block
* #param block
* #return
* #throws NoSuchAlgorithmException
*/
public static boolean is_valid_block(Block last_block, Block block) throws NoSuchAlgorithmException {
String binary_hash = CryptoHash.hex_to_binary(block.getHash());
char[] pow_array = CryptoHash.n_len_array('0', block.getDifficulty());
char[] binary_char_array = CryptoHash.string_to_charray(binary_hash);
if (!block.getLastHash().equalsIgnoreCase(last_block.getHash())) {
System.out.println("The last hash must be correct");
return false;
// Throw exception the last hash must be correct
}
if (!Arrays.equals(pow_array, Arrays.copyOfRange(binary_char_array, 0, block.getDifficulty()))) {
System.out.println("Proof of work requirement not met");
return false;
// throw exception - proof of work requirement not met
}
if (Math.abs(last_block.difficulty - block.difficulty) > 1) {
System.out.println("Block difficulty must adjust by one");
return false;
// throw exception: The block difficulty must only adjust by 1
}
String reconstructed_hash = CryptoHash.getSHA256(block.getTimestamp(), block.getLastHash(), block.getData(),
block.getDifficulty(), block.getNonce());
if (!block.getHash().equalsIgnoreCase(reconstructed_hash)) {
System.out.println("The block hash must be correct");
System.out.println(block.getHash());
System.out.println(reconstructed_hash);
return false;
// throw exception: the block hash must be correct
}
System.out.println("You have mined a valid block");
return true;
}
public int getDifficulty() {
return difficulty;
}
public long getTimestamp() {
return timestamp;
}
public String getHash() {
return hash;
}
public String getLastHash() {
return lastHash;
}
public String[] getData() {
return data;
}
public int getNonce() {
return nonce;
}
public static void main(String[] args) throws NoSuchAlgorithmException {
// String md = CryptoHash.getSHA256("foobar");
Block genesis = genesis_block();
System.out.println(genesis.toString());
// Block bad_block = Block.mine_block(genesis, new String[] { "watch", "AOT" });
// bad_block.lastHash = "evil data";
// System.out.println(bad_block.toString());
Block good_block = mine_block(genesis, new String[] { "foo", "bar" });
System.out.println(good_block.toString());
// System.out.println(mine_block(new_block, new String[] { "crypto", "is", "fun" }).toString());
// System.out.println(Block.is_valid_block(genesis, bad_block)); // returns false as expected
System.out.println(Block.is_valid_block(genesis, good_block));
System.out.println(CryptoHash.hex_to_binary(good_block.getHash()));
Block good_block2 = mine_block(good_block, new String[] { "bar", "foo" });
Block good_block3 = mine_block(good_block2, new String[] { "bar", "foo" });
Block good_block4 = mine_block(good_block3, new String[] { "bar", "foo" });
// Block good_block5 = mine_block(good_block4, new String[] {"bar", "foo"});
// Block good_block6 = mine_block(good_block5, new String[] {"bar", "foo"});
}
}
package privblock.gerald.ryan.util;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.HashMap;
public class CryptoHash {
static HashMap<Character, String> HEX_TO_BIN_TABLE;
static {
HEX_TO_BIN_TABLE = new HashMap<Character, String>();
HEX_TO_BIN_TABLE.put('0', "0000");
HEX_TO_BIN_TABLE.put('1', "0001");
HEX_TO_BIN_TABLE.put('2', "0010");
HEX_TO_BIN_TABLE.put('3', "0011");
HEX_TO_BIN_TABLE.put('4', "0100");
HEX_TO_BIN_TABLE.put('5', "0101");
HEX_TO_BIN_TABLE.put('6', "0110");
HEX_TO_BIN_TABLE.put('7', "0111");
HEX_TO_BIN_TABLE.put('8', "1000");
HEX_TO_BIN_TABLE.put('9', "1001");
HEX_TO_BIN_TABLE.put('a', "1010");
HEX_TO_BIN_TABLE.put('b', "1011");
HEX_TO_BIN_TABLE.put('c', "1100");
HEX_TO_BIN_TABLE.put('d', "1101");
HEX_TO_BIN_TABLE.put('e', "1110");
HEX_TO_BIN_TABLE.put('f', "1111");
}
public static String getSHA256(String... sarray) throws NoSuchAlgorithmException {
String s = concat(sarray);
// System.out.printf("Hashing \"%s\"\n", s);
MessageDigest md;
md = MessageDigest.getInstance("SHA-256");
byte[] b = md.digest(s.getBytes(StandardCharsets.UTF_8));
BigInteger number = new BigInteger(1, b);
StringBuilder hexString = new StringBuilder(number.toString(16));
while (hexString.length() < 32) {
hexString.insert(0, '0');
}
String mds = hexString.toString();
// System.out.printf("hash is:\n%s\n", mds);
return hexString.toString();
}
public static String getSHA256(long timestamp, String last_hash, String[] data, int difficulty, int nonce)
throws NoSuchAlgorithmException {
String s = "";
s += Long.toString(timestamp);
s += last_hash;
s += concat(data);
s += Integer.toString(difficulty);
s += Integer.toString(nonce);
// System.out.printf("Hashing \"%s\"\n", s);
MessageDigest md;
md = MessageDigest.getInstance("SHA-256");
byte[] b = md.digest(s.getBytes(StandardCharsets.UTF_8));
BigInteger number = new BigInteger(1, b);
StringBuilder hexString = new StringBuilder(number.toString(16));
// System.out.println(hexString);
while (hexString.length() < 32) {
hexString.insert(0, '0');
}
String messageDigestString = hexString.toString();
// System.out.printf("hash is:\n%s\n", messageDigestString);
return hexString.toString();
}
public static char[] n_len_array(char c, int n) {
char[] ch = new char[n];
for (int i = 0; i<n; i++) {
ch[i] = c;
}
return ch;
}
public static String n_len_string(char c, int n) {
String s = "";
for (int i = 0; i<n; i++) {
s += c;
}
return s;
}
public static String concat(String... args) {
String s = "";
for (String $ : args) {
s += $;
}
// System.out.println(s);
return s;
}
public static char[] string_to_charray(String str) {
char[] ch = new char[str.length()];
for (int i = 0; i < str.length(); i++) {
ch[i] = str.charAt(i);
}
return ch;
}
public static String string_to_hex(String arg) {
return String.format("%064x", new BigInteger(1, arg.getBytes(StandardCharsets.UTF_8)));
}
public static String hex_to_binary(String hex_string) {
String binary_string = "";
for (int i = 0; i < hex_string.length(); i++) {
binary_string += HEX_TO_BIN_TABLE.get(hex_string.charAt(i));
}
return binary_string;
}
public static String string_to_binary(String raw_string) {
String hex_string = string_to_hex(raw_string);
String bin_string = hex_to_binary(hex_string);
return bin_string;
}
}
ps here's an example of a log I created. I created other cleaner logs too but this shows what we're working with. The first item represents time in milliseconds. The second represents the first four digits of the hash, which is directly below it, followed by the level of difficulty requirement string (what the second item needs to be, length n = difficulty level). The hash just never leads with four zeros, ever, so my hash function or call to the function must be broken in some way.
6479
1000
1000001010111011100110111010100100111010101001111110010101011101101101110000110100110110110000001010001000000010110001100111100111010100110001001001110111011010011100110000011111110100000100000100000010100001000110000111000101100010001111011000110011111101
0000
6479
0101
0101110111010100101010100000001011100011000001110001011011001101001111101011010011000111101101111111001001001010100110101101100111111011001011100101111000011100010001000000000011000111010000101101001000001010101010111001010000101001110011111101011011011000
0000
6479
1000
1000000001000101001110001110110000110111001101100001011000111010111110001011011010011111111101011001110011001001111011011110110010101010101100011011001001110001100010010101001011100001101011011101010000000100111100011011110100000101100111010100100110011101
0000
6479
I figured out the problem. It is indeed often returning 4 leading zeroes but the code as structured is clipping them off (because it doesn't think they have meaning). I noticed by logging that the length is not always a fixed 64byte/256 bit string. Here's the output:
256
1101111000010000100001110001010001010000001010111001100011010011110010001001010001010010100110111000110010000010001110110100100101000000001111111110011100000001010100000111001000111101010001010100110100000000111000100001000000010010010111011110110011110111
256
011001111101001000011111011001111110010110000011001011111010001011010110010100001011010011010010111101100010010111000010110010110111110001010101100000000101001000111110100111011100001110010010101011011000000101100001101110101101010001110000111111110000
252
0001100101110011101000000011000101011100111101110100111110100101110110011100010110001011000110010011110110011001100111010001100100011001011000001011100011011011011011101000111000011100100011011011011000101010011101000110101011000110011100111010000011000011
256
1100110001001001110001100111100010101100100010110111100111001010011011111111100010100110110000010000101000010111111010010101110001100010101010111111111111001011010111010100001010000010111100100100111000010101011000110000100000100111010001000011000000010000
256
So that's solved, or at least I understand the problem. It's amazing what sleep will do.
I wanted to mock a static method used inside a private method of a class and return a specific value.
public class Test {
private String encodeValue(String abc) {
try {
return URLEncoder.encode(...);
} catch (UnsupportedEncodingException e) {
throw InvalidValueException.create("Error converting");
}
}
URLEncoder.encode ->encode is a static method inside URLEncoder.
In Test class using powermock works:
PowerMock.mockStatic(URLEncoder.class);
expect(URLEncoder.encode()).andThrow(new UnsupportedEncodingException());
PowerMock.replay(URLEncoder.class);
String encoded = Whitebox.invokeMethod(testMock,"encodeVaue","Apple Mango");
But i wanted to replace Powermock with any other mocking ways available.
Is there a way to mock the above class.
URL Encoder class:
/**
* Translates a string into {#code application/x-www-form-urlencoded}
* format using a specific encoding scheme. This method uses the
* supplied encoding scheme to obtain the bytes for unsafe
* characters.
* <p>
* <em><strong>Note:</strong> The <a href=
* "http://www.w3.org/TR/html40/appendix/notes.html#non-ascii-chars">
* World Wide Web Consortium Recommendation</a> states that
* UTF-8 should be used. Not doing so may introduce
* incompatibilities.</em>
*
* #param s {#code String} to be translated.
* #param enc The name of a supported
* <a href="../lang/package-summary.html#charenc">character
* encoding</a>.
* #return the translated {#code String}.
* #exception UnsupportedEncodingException
* If the named encoding is not supported
* #see URLDecoder#decode(java.lang.String, java.lang.String)
* #since 1.4
*/
public static String encode(String s, String enc)
throws UnsupportedEncodingException {
boolean needToChange = false;
StringBuffer out = new StringBuffer(s.length());
Charset charset;
CharArrayWriter charArrayWriter = new CharArrayWriter();
if (enc == null)
throw new NullPointerException("charsetName");
try {
charset = Charset.forName(enc);
} catch (IllegalCharsetNameException e) {
throw new UnsupportedEncodingException(enc);
} catch (UnsupportedCharsetException e) {
throw new UnsupportedEncodingException(enc);
}
for (int i = 0; i < s.length();) {
int c = (int) s.charAt(i);
//System.out.println("Examining character: " + c);
if (dontNeedEncoding.get(c)) {
if (c == ' ') {
c = '+';
needToChange = true;
}
//System.out.println("Storing: " + c);
out.append((char)c);
i++;
} else {
// convert to external encoding before hex conversion
do {
charArrayWriter.write(c);
/*
* If this character represents the start of a Unicode
* surrogate pair, then pass in two characters. It's not
* clear what should be done if a bytes reserved in the
* surrogate pairs range occurs outside of a legal
* surrogate pair. For now, just treat it as if it were
* any other character.
*/
if (c >= 0xD800 && c <= 0xDBFF) {
/*
System.out.println(Integer.toHexString(c)
+ " is high surrogate");
*/
if ( (i+1) < s.length()) {
int d = (int) s.charAt(i+1);
/*
System.out.println("\tExamining "
+ Integer.toHexString(d));
*/
if (d >= 0xDC00 && d <= 0xDFFF) {
/*
System.out.println("\t"
+ Integer.toHexString(d)
+ " is low surrogate");
*/
charArrayWriter.write(d);
i++;
}
}
}
i++;
} while (i < s.length() && !dontNeedEncoding.get((c = (int) s.charAt(i))));
charArrayWriter.flush();
String str = new String(charArrayWriter.toCharArray());
byte[] ba = str.getBytes(charset);
for (int j = 0; j < ba.length; j++) {
out.append('%');
char ch = Character.forDigit((ba[j] >> 4) & 0xF, 16);
// converting to use uppercase letter as part of
// the hex value if ch is a letter.
if (Character.isLetter(ch)) {
ch -= caseDiff;
}
out.append(ch);
ch = Character.forDigit(ba[j] & 0xF, 16);
if (Character.isLetter(ch)) {
ch -= caseDiff;
}
out.append(ch);
}
charArrayWriter.reset();
needToChange = true;
}
}
return (needToChange? out.toString() : s);
}
Mocking privates and statics is one of the chief strengths of JMockit over other mocking frameworks.
The class you call "Test" is really the "ClassUnderTest", so apologies, but the test of "Test" is "TestTest" :)
public class TestTest {
#Tested
public Test cut;
#Test
public void testencodeValue() {
// Mock the static
new MockUp<URLEncoder>() {
#Mock
String encode(String s, String enc) {
return "JMockit FTW";
}
};
// invoke the private method
final Method method = MethodReflection.findCompatibleMethod(Test.class, "encodeValue", new Class<?>[] { String.class });
final String res = MethodReflection.invoke(cut, method);
assertEquals("JMockit FTW", res);
}
}
That said, testing privates is sort of a PITA. I am generally of the mind that if a method is worth testing, it is almost certainly worth exposing. The same criteria that make the method worth testing means that somebody-somewhere-someday will want to override your implementation and provide a slightly alternative one. Make their job easy, and make it protected. Make your (testing) job easy, and do the same thing.
How can you play multiple (audio) byte arrays simultaneously? This "byte array" is recorded by TargetDataLine, transferred using a server.
What I've tried so far
Using SourceDataLine:
There is no way to play mulitple streams using SourceDataLine, because the write method blocks until the buffer is written. This problem cannot be fixed using Threads, because only one SourceDataLine can write concurrently.
Using the AudioPlayer Class:
ByteInputStream stream2 = new ByteInputStream(data, 0, data.length);
AudioInputStream stream = new AudioInputStream(stream2, VoiceChat.format, data.length);
AudioPlayer.player.start(stream);
This just plays noise on the clients.
EDIT
I don't receive the voice packets at the same time, it's not simultaneously, more "overlapping".
Apparently Java's Mixer interface was not designed for this.
http://docs.oracle.com/javase/7/docs/api/javax/sound/sampled/Mixer.html:
A mixer is an audio device with one or more lines. It need not be
designed for mixing audio signals.
And indeed, when I try to open multiple lines on the same mixer this fails with a LineUnavailableException. However if all your audio recordings have the same audio format it's quite easy to manually mix them together. For example if you have 2 inputs:
Convert both to the appropriate data type (for example byte[] for 8 bit audio, short[] for 16 bit, float[] for 32 bit floating point etc)
Sum them in another array. Make sure summed values do not exceed the range of the datatype.
Convert output back to bytes and write that to the SourceDataLine
See also How is audio represented with numbers?
Here's a sample mixing down 2 recordings and outputting as 1 signal, all in 16bit 48Khz stereo.
// print all devices (both input and output)
int i = 0;
Mixer.Info[] infos = AudioSystem.getMixerInfo();
for (Mixer.Info info : infos)
System.out.println(i++ + ": " + info.getName());
// select 2 inputs and 1 output
System.out.println("Select input 1: ");
int in1Index = Integer.parseInt(System.console().readLine());
System.out.println("Select input 2: ");
int in2Index = Integer.parseInt(System.console().readLine());
System.out.println("Select output: ");
int outIndex = Integer.parseInt(System.console().readLine());
// ugly java sound api stuff
try (Mixer in1Mixer = AudioSystem.getMixer(infos[in1Index]);
Mixer in2Mixer = AudioSystem.getMixer(infos[in2Index]);
Mixer outMixer = AudioSystem.getMixer(infos[outIndex])) {
in1Mixer.open();
in2Mixer.open();
outMixer.open();
try (TargetDataLine in1Line = (TargetDataLine) in1Mixer.getLine(in1Mixer.getTargetLineInfo()[0]);
TargetDataLine in2Line = (TargetDataLine) in2Mixer.getLine(in2Mixer.getTargetLineInfo()[0]);
SourceDataLine outLine = (SourceDataLine) outMixer.getLine(outMixer.getSourceLineInfo()[0])) {
// audio format 48khz 16 bit stereo (signed litte endian)
AudioFormat format = new AudioFormat(48000.0f, 16, 2, true, false);
// 4 bytes per frame (16 bit samples stereo)
int frameSize = 4;
int bufferSize = 4800;
int bufferBytes = frameSize * bufferSize;
// buffers for java audio
byte[] in1Bytes = new byte[bufferBytes];
byte[] in2Bytes = new byte[bufferBytes];
byte[] outBytes = new byte[bufferBytes];
// buffers for mixing
short[] in1Samples = new short[bufferBytes / 2];
short[] in2Samples = new short[bufferBytes / 2];
short[] outSamples = new short[bufferBytes / 2];
// how long to record & play
int framesProcessed = 0;
int durationSeconds = 10;
int durationFrames = (int) (durationSeconds * format.getSampleRate());
// open devices
in1Line.open(format, bufferBytes);
in2Line.open(format, bufferBytes);
outLine.open(format, bufferBytes);
in1Line.start();
in2Line.start();
outLine.start();
// start audio loop
while (framesProcessed < durationFrames) {
// record audio
in1Line.read(in1Bytes, 0, bufferBytes);
in2Line.read(in2Bytes, 0, bufferBytes);
// convert input bytes to samples
ByteBuffer.wrap(in1Bytes).order(ByteOrder.LITTLE_ENDIAN).asShortBuffer().get(in1Samples);
ByteBuffer.wrap(in2Bytes).order(ByteOrder.LITTLE_ENDIAN).asShortBuffer().get(in2Samples);
// mix samples - lower volume by 50% since we're mixing 2 streams
for (int s = 0; s < bufferBytes / 2; s++)
outSamples[s] = (short) ((in1Samples[s] + in2Samples[s]) * 0.5);
// convert output samples to bytes
ByteBuffer.wrap(outBytes).order(ByteOrder.LITTLE_ENDIAN).asShortBuffer().put(outSamples);
// play audio
outLine.write(outBytes, 0, bufferBytes);
framesProcessed += bufferBytes / frameSize;
}
in1Line.stop();
in2Line.stop();
outLine.stop();
}
}
Allright, I put something together which should get you started. I'll post the full code below but I'll first try and explain the steps involved.
The interesting part here is to create you're own audio "mixer" class which allows consumers of that class to schedule audio blocks at specific points in the (near) future. The specific-point-in-time part is important here: i'm assuming you receive network voices in packets where each packet needs to start exactly at the end of the previous one in order to play back a continuous sound for a single voice. Also since you say voices can overlap I'm assuming (yes, lots of assumptions) a new one can come in over the network while one or more old ones are still playing. So it seems reasonable to allow audio blocks to be scheduled from any thread. Note that there's only one thread actually writing to the dataline, it's just that any thread can submit audio packets to the mixer.
So for the submit-audio-packet part we now have this:
private final ConcurrentLinkedQueue<QueuedBlock> scheduledBlocks;
public void mix(long when, short[] block) {
scheduledBlocks.add(new QueuedBlock(when, Arrays.copyOf(block, block.length)));
}
The QueuedBlock class is just used to tag a byte array (the audio buffer) with the "when": the point in time where the block should be played.
Points in time are expressed relative to the current position of the audio stream. It is set to zero when the stream is created and updated with the buffer size each time an audio buffer is written to the dataline:
private final AtomicLong position = new AtomicLong();
public long position() {
return position.get();
}
Apart from all the hassle to set up the data line, the interesting part of the mixer class is obviously where the mixdown happens. For each scheduled audio block, it's split up into 3 cases:
The block is already played in it's entirety. Remove from the scheduledBlocks list.
The block is scheduled to start at some point in time after the current buffer. Do nothing.
(Part of) the block should be mixed down into the current buffer. Note that the beginning of the block may (or may not) be already played in previous buffer(s). Similarly, the end of the scheduled block may exceed the end of the current buffer in which case we mix down the first part of it and leave the rest for the next round, untill all of it has been played an the entire block is removed.
Also note that there's no reliable way to start playing audio data immediately, when you submit packets to the mixer be sure to always have them start at least the duration of 1 audio buffer from now otherwise you'll risk losing the beginning of your sound. Here's the mixdown code:
private static final double MIXDOWN_VOLUME = 1.0 / NUM_PRODUCERS;
private final List<QueuedBlock> finished = new ArrayList<>();
private final short[] mixBuffer = new short[BUFFER_SIZE_FRAMES * CHANNELS];
private final byte[] audioBuffer = new byte[BUFFER_SIZE_FRAMES * CHANNELS * 2];
private final AtomicLong position = new AtomicLong();
Arrays.fill(mixBuffer, (short) 0);
long bufferStartAt = position.get();
for (QueuedBlock block : scheduledBlocks) {
int blockFrames = block.data.length / CHANNELS;
// block fully played - mark for deletion
if (block.when + blockFrames <= bufferStartAt) {
finished.add(block);
continue;
}
// block starts after end of current buffer
if (bufferStartAt + BUFFER_SIZE_FRAMES <= block.when)
continue;
// mix in part of the block which overlaps current buffer
int blockOffset = Math.max(0, (int) (bufferStartAt - block.when));
int blockMaxFrames = blockFrames - blockOffset;
int bufferOffset = Math.max(0, (int) (block.when - bufferStartAt));
int bufferMaxFrames = BUFFER_SIZE_FRAMES - bufferOffset;
for (int f = 0; f < blockMaxFrames && f < bufferMaxFrames; f++)
for (int c = 0; c < CHANNELS; c++) {
int bufferIndex = (bufferOffset + f) * CHANNELS + c;
int blockIndex = (blockOffset + f) * CHANNELS + c;
mixBuffer[bufferIndex] += (short)
(block.data[blockIndex]*MIXDOWN_VOLUME);
}
}
scheduledBlocks.removeAll(finished);
finished.clear();
ByteBuffer
.wrap(audioBuffer)
.order(ByteOrder.LITTLE_ENDIAN)
.asShortBuffer()
.put(mixBuffer);
line.write(audioBuffer, 0, audioBuffer.length);
position.addAndGet(BUFFER_SIZE_FRAMES);
And finally a complete, self-contained sample which spawns a number of threads submitting audio blocks representing sinewaves of random duration and frequency to the mixer (called AudioConsumer in this sample). Replace sinewaves by incoming network packets and you should be halfway to a solution.
package test;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import javax.sound.sampled.AudioFormat;
import javax.sound.sampled.AudioSystem;
import javax.sound.sampled.Line;
import javax.sound.sampled.Mixer;
import javax.sound.sampled.SourceDataLine;
public class Test {
public static final int CHANNELS = 2;
public static final int SAMPLE_RATE = 48000;
public static final int NUM_PRODUCERS = 10;
public static final int BUFFER_SIZE_FRAMES = 4800;
// generates some random sine wave
public static class ToneGenerator {
private static final double[] NOTES = {261.63, 311.13, 392.00};
private static final double[] OCTAVES = {1.0, 2.0, 4.0, 8.0};
private static final double[] LENGTHS = {0.05, 0.25, 1.0, 2.5, 5.0};
private double phase;
private int framesProcessed;
private final double length;
private final double frequency;
public ToneGenerator() {
ThreadLocalRandom rand = ThreadLocalRandom.current();
length = LENGTHS[rand.nextInt(LENGTHS.length)];
frequency = NOTES[rand.nextInt(NOTES.length)] * OCTAVES[rand.nextInt(OCTAVES.length)];
}
// make sound
public void fill(short[] block) {
for (int f = 0; f < block.length / CHANNELS; f++) {
double sample = Math.sin(phase * 2.0 * Math.PI);
for (int c = 0; c < CHANNELS; c++)
block[f * CHANNELS + c] = (short) (sample * Short.MAX_VALUE);
phase += frequency / SAMPLE_RATE;
}
framesProcessed += block.length / CHANNELS;
}
// true if length of tone has been generated
public boolean done() {
return framesProcessed >= length * SAMPLE_RATE;
}
}
// dummy audio producer, based on sinewave generator
// above but could also be incoming network packets
public static class AudioProducer {
final Thread thread;
final AudioConsumer consumer;
final short[] buffer = new short[BUFFER_SIZE_FRAMES * CHANNELS];
public AudioProducer(AudioConsumer consumer) {
this.consumer = consumer;
thread = new Thread(() -> run());
thread.setDaemon(true);
}
public void start() {
thread.start();
}
// repeatedly play random sine and sleep for some time
void run() {
try {
ThreadLocalRandom rand = ThreadLocalRandom.current();
while (true) {
long pos = consumer.position();
ToneGenerator g = new ToneGenerator();
// if we schedule at current buffer position, first part of the tone will be
// missed so have tone start somewhere in the middle of the next buffer
pos += BUFFER_SIZE_FRAMES + rand.nextInt(BUFFER_SIZE_FRAMES);
while (!g.done()) {
g.fill(buffer);
consumer.mix(pos, buffer);
pos += BUFFER_SIZE_FRAMES;
// we can generate audio faster than it's played
// sleep a while to compensate - this more closely
// corresponds to playing audio coming in over the network
double bufferLengthMillis = BUFFER_SIZE_FRAMES * 1000.0 / SAMPLE_RATE;
Thread.sleep((int) (bufferLengthMillis * 0.9));
}
// sleep a while in between tones
Thread.sleep(1000 + rand.nextInt(2000));
}
} catch (Throwable t) {
System.out.println(t.getMessage());
t.printStackTrace();
}
}
}
// audio consumer - plays continuously on a background
// thread, allows audio to be mixed in from arbitrary threads
public static class AudioConsumer {
// audio block with "when to play" tag
private static class QueuedBlock {
final long when;
final short[] data;
public QueuedBlock(long when, short[] data) {
this.when = when;
this.data = data;
}
}
// need not normally be so low but in this example
// we're mixing down a bunch of full scale sinewaves
private static final double MIXDOWN_VOLUME = 1.0 / NUM_PRODUCERS;
private final List<QueuedBlock> finished = new ArrayList<>();
private final short[] mixBuffer = new short[BUFFER_SIZE_FRAMES * CHANNELS];
private final byte[] audioBuffer = new byte[BUFFER_SIZE_FRAMES * CHANNELS * 2];
private final Thread thread;
private final AtomicLong position = new AtomicLong();
private final AtomicBoolean running = new AtomicBoolean(true);
private final ConcurrentLinkedQueue<QueuedBlock> scheduledBlocks = new ConcurrentLinkedQueue<>();
public AudioConsumer() {
thread = new Thread(() -> run());
}
public void start() {
thread.start();
}
public void stop() {
running.set(false);
}
// gets the play cursor. note - this is not accurate and
// must only be used to schedule blocks relative to other blocks
// (e.g., for splitting up continuous sounds into multiple blocks)
public long position() {
return position.get();
}
// put copy of audio block into queue so we don't
// have to worry about caller messing with it afterwards
public void mix(long when, short[] block) {
scheduledBlocks.add(new QueuedBlock(when, Arrays.copyOf(block, block.length)));
}
// better hope mixer 0, line 0 is output
private void run() {
Mixer.Info[] mixerInfo = AudioSystem.getMixerInfo();
try (Mixer mixer = AudioSystem.getMixer(mixerInfo[0])) {
Line.Info[] lineInfo = mixer.getSourceLineInfo();
try (SourceDataLine line = (SourceDataLine) mixer.getLine(lineInfo[0])) {
line.open(new AudioFormat(SAMPLE_RATE, 16, CHANNELS, true, false), BUFFER_SIZE_FRAMES);
line.start();
while (running.get())
processSingleBuffer(line);
line.stop();
}
} catch (Throwable t) {
System.out.println(t.getMessage());
t.printStackTrace();
}
}
// mix down single buffer and offer to the audio device
private void processSingleBuffer(SourceDataLine line) {
Arrays.fill(mixBuffer, (short) 0);
long bufferStartAt = position.get();
// mixdown audio blocks
for (QueuedBlock block : scheduledBlocks) {
int blockFrames = block.data.length / CHANNELS;
// block fully played - mark for deletion
if (block.when + blockFrames <= bufferStartAt) {
finished.add(block);
continue;
}
// block starts after end of current buffer
if (bufferStartAt + BUFFER_SIZE_FRAMES <= block.when)
continue;
// mix in part of the block which overlaps current buffer
// note that block may have already started in the past
// but extends into the current buffer, or that it starts
// in the future but before the end of the current buffer
int blockOffset = Math.max(0, (int) (bufferStartAt - block.when));
int blockMaxFrames = blockFrames - blockOffset;
int bufferOffset = Math.max(0, (int) (block.when - bufferStartAt));
int bufferMaxFrames = BUFFER_SIZE_FRAMES - bufferOffset;
for (int f = 0; f < blockMaxFrames && f < bufferMaxFrames; f++)
for (int c = 0; c < CHANNELS; c++) {
int bufferIndex = (bufferOffset + f) * CHANNELS + c;
int blockIndex = (blockOffset + f) * CHANNELS + c;
mixBuffer[bufferIndex] += (short) (block.data[blockIndex] * MIXDOWN_VOLUME);
}
}
scheduledBlocks.removeAll(finished);
finished.clear();
ByteBuffer.wrap(audioBuffer).order(ByteOrder.LITTLE_ENDIAN).asShortBuffer().put(mixBuffer);
line.write(audioBuffer, 0, audioBuffer.length);
position.addAndGet(BUFFER_SIZE_FRAMES);
}
}
public static void main(String[] args) {
System.out.print("Press return to exit...");
AudioConsumer consumer = new AudioConsumer();
consumer.start();
for (int i = 0; i < NUM_PRODUCERS; i++)
new AudioProducer(consumer).start();
System.console().readLine();
consumer.stop();
}
}
You can use the Tritontus library to do software audio mixing (it's old but still works quite well).
Add the dependency to your project:
<dependency>
<groupId>com.googlecode.soundlibs</groupId>
<artifactId>tritonus-all</artifactId>
<version>0.3.7.2</version>
</dependency>
Use the org.tritonus.share.sampled.FloatSampleBuffer. Both buffers must be of same AudioFormat before calling #mix.
// TODO instantiate these variables with real data
byte[] audio1, audio2;
AudioFormat af1, af2;
SourceDataLine sdl = AudioSystem.getSourceDataLine(af1);
FloatSampleBuffer fsb1 = new FloatSampleBuffer(audio1, 0, audio1.length, af1.getFormat());
FloatSampleBuffer fsb2 = new FloatSampleBuffer(audio2, 0, audio2.length, af2.getFormat());
fsb1.mix(fsb2);
byte[] result = fsb1.convertToByteArray(af1);
sdl.write(result, 0, result.length); // play it
Hi Team, I am trying to find a String "Henry" in a binary file and change the String to a different string. FYI the file is the output of serialisation of an object. Original Question here
I am new to searching bytes and imagined this code would search for my byte[] and exchange it. But it doesn't come close to working it doesn't even find a match.
{
byte[] bytesHenry = new String("Henry").getBytes();
byte[] bytesSwap = new String("Zsswd").getBytes();
byte[] seekHenry = new byte[bytesHenry.length];
RandomAccessFile file = new RandomAccessFile(fileString,"rw");
long filePointer;
while (seekHenry != null) {
filePointer = file.getFilePointer();
file.readFully(seekHenry);
if (bytesHenry == seekHenry) {
file.seek(filePointer);
file.write(bytesSwap);
break;
}
}
}
Okay I see the bytesHenry==seekHenry problem and will swap to Arrays.equals( bytesHenry , seekHenry )
I think I need to move along by -4 byte positions each time i read 5 bytes.
Bingo it finds it now
while (seekHenry != null) {
filePointer = file.getFilePointer();
file.readFully(seekHenry);;
if (Arrays.equals(bytesHenry,
seekHenry)) {
file.seek(filePointer);
file.write(bytesSwap);
break;
}
file.seek(filePointer);
file.read();
}
The following could work for you, see the method search(byte[] input, byte[] searchedFor) which returns the index where the first match matches, or -1.
public class SearchBuffer {
public static void main(String[] args) throws UnsupportedEncodingException {
String charset= "US-ASCII";
byte[] searchedFor = "ciao".getBytes(charset);
byte[] input = "aaaciaaaciaojjcia".getBytes(charset);
int idx = search(input, searchedFor);
System.out.println("index: "+idx); //should be 8
}
public static int search(byte[] input, byte[] searchedFor) {
//convert byte[] to Byte[]
Byte[] searchedForB = new Byte[searchedFor.length];
for(int x = 0; x<searchedFor.length; x++){
searchedForB[x] = searchedFor[x];
}
int idx = -1;
//search:
Deque<Byte> q = new ArrayDeque<Byte>(input.length);
for(int i=0; i<input.length; i++){
if(q.size() == searchedForB.length){
//here I can check
Byte[] cur = q.toArray(new Byte[]{});
if(Arrays.equals(cur, searchedForB)){
//found!
idx = i - searchedForB.length;
break;
} else {
//not found
q.pop();
q.addLast(input[i]);
}
} else {
q.addLast(input[i]);
}
}
return idx;
}
}
From Fastest way to find a string in a text file with java:
The best realization I've found in MIMEParser: https://github.com/samskivert/ikvm-openjdk/blob/master/build/linux-amd64/impsrc/com/sun/xml/internal/org/jvnet/mimepull/MIMEParser.java
/**
* Finds the boundary in the given buffer using Boyer-Moore algo.
* Copied from java.util.regex.Pattern.java
*
* #param mybuf boundary to be searched in this mybuf
* #param off start index in mybuf
* #param len number of bytes in mybuf
*
* #return -1 if there is no match or index where the match starts
*/
private int match(byte[] mybuf, int off, int len) {
Needed also:
private void compileBoundaryPattern();
I would like to create a version of Java's deflater not synchronizing on its ZStreamRef instance.
I started by copying the code into a Deflater2 class. The code compiles, however, when I create an instance of that class, I get:
Exception in thread "main" java.lang.UnsatisfiedLinkError: net.dwst.bricolo.ResetGZip.Deflater2.initIDs()V
at net.mypackage.Deflater2.initIDs(Native Method)
at net.mypackage.Deflater2.<clinit>(Deflater2.java:65)
at net.mypackage.SpeedTest.main(SpeedTest.java:13)
How can I create an instance of Deflater2 without triggering this exception?
I am providing the code for Deflater2:
package net.mypackage;
public class Deflater2 {
private final ZStreamRef zsRef;
private byte[] buf = new byte[0];
private int off, len;
private int level, strategy;
private boolean setParams;
private boolean finish, finished;
/**
* Compression method for the deflate algorithm (the only one currently
* supported).
*/
public static final int DEFLATED = 8;
/**
* Compression level for no compression.
*/
public static final int NO_COMPRESSION = 0;
/**
* Compression level for fastest compression.
*/
public static final int BEST_SPEED = 1;
/**
* Compression level for best compression.
*/
public static final int BEST_COMPRESSION = 9;
/**
* Default compression level.
*/
public static final int DEFAULT_COMPRESSION = -1;
/**
* Compression strategy best used for data consisting mostly of small
* values with a somewhat random distribution. Forces more Huffman coding
* and less string matching.
*/
public static final int FILTERED = 1;
/**
* Compression strategy for Huffman coding only.
*/
public static final int HUFFMAN_ONLY = 2;
/**
* Default compression strategy.
*/
public static final int DEFAULT_STRATEGY = 0;
static {
/* Zip library is loaded from System.initializeSystemClass */
initIDs();
}
/**
* Creates a new compressor using the specified compression level.
* If 'nowrap' is true then the ZLIB header and checksum fields will
* not be used in order to support the compression format used in
* both GZIP and PKZIP.
* #param level the compression level (0-9)
* #param nowrap if true then use GZIP compatible compression
*/
public Deflater2(int level, boolean nowrap) {
this.level = level;
this.strategy = DEFAULT_STRATEGY;
this.zsRef = new ZStreamRef(init(level, DEFAULT_STRATEGY, nowrap));
}
/**
* Creates a new compressor using the specified compression level.
* Compressed data will be generated in ZLIB format.
* #param level the compression level (0-9)
*/
public Deflater2(int level) {
this(level, false);
}
/**
* Creates a new compressor with the default compression level.
* Compressed data will be generated in ZLIB format.
*/
public Deflater2() {
this(DEFAULT_COMPRESSION, false);
}
/**
* Sets input data for compression. This should be called whenever
* needsInput() returns true indicating that more input data is required.
* #param b the input data bytes
* #param off the start offset of the data
* #param len the length of the data
* #see Deflater#needsInput
*/
public void setInput(byte[] b, int off, int len) {
if (b== null) {
throw new NullPointerException();
}
if (off < 0 || len < 0 || off > b.length - len) {
throw new ArrayIndexOutOfBoundsException();
}
synchronized (zsRef) {
this.buf = b;
this.off = off;
this.len = len;
}
}
/**
* Sets input data for compression. This should be called whenever
* needsInput() returns true indicating that more input data is required.
* #param b the input data bytes
* #see Deflater#needsInput
*/
public void setInput(byte[] b) {
setInput(b, 0, b.length);
}
/**
* Sets preset dictionary for compression. A preset dictionary is used
* when the history buffer can be predetermined. When the data is later
* uncompressed with Inflater.inflate(), Inflater.getAdler() can be called
* in order to get the Adler-32 value of the dictionary required for
* decompression.
* #param b the dictionary data bytes
* #param off the start offset of the data
* #param len the length of the data
* #see Inflater#inflate
* #see Inflater#getAdler
*/
public void setDictionary(byte[] b, int off, int len) {
if (b == null) {
throw new NullPointerException();
}
if (off < 0 || len < 0 || off > b.length - len) {
throw new ArrayIndexOutOfBoundsException();
}
synchronized (zsRef) {
ensureOpen();
setDictionary(zsRef.address(), b, off, len);
}
}
/**
* Sets preset dictionary for compression. A preset dictionary is used
* when the history buffer can be predetermined. When the data is later
* uncompressed with Inflater.inflate(), Inflater.getAdler() can be called
* in order to get the Adler-32 value of the dictionary required for
* decompression.
* #param b the dictionary data bytes
* #see Inflater#inflate
* #see Inflater#getAdler
*/
public void setDictionary(byte[] b) {
setDictionary(b, 0, b.length);
}
/**
* Sets the compression strategy to the specified value.
* #param strategy the new compression strategy
* #exception IllegalArgumentException if the compression strategy is
* invalid
*/
public void setStrategy(int strategy) {
switch (strategy) {
case DEFAULT_STRATEGY:
case FILTERED:
case HUFFMAN_ONLY:
break;
default:
throw new IllegalArgumentException();
}
synchronized (zsRef) {
if (this.strategy != strategy) {
this.strategy = strategy;
setParams = true;
}
}
}
/**
* Sets the current compression level to the specified value.
* #param level the new compression level (0-9)
* #exception IllegalArgumentException if the compression level is invalid
*/
public void setLevel(int level) {
if ((level < 0 || level > 9) && level != DEFAULT_COMPRESSION) {
throw new IllegalArgumentException("invalid compression level");
}
synchronized (zsRef) {
if (this.level != level) {
this.level = level;
setParams = true;
}
}
}
/**
* Returns true if the input data buffer is empty and setInput()
* should be called in order to provide more input.
* #return true if the input data buffer is empty and setInput()
* should be called in order to provide more input
*/
public boolean needsInput() {
return len <= 0;
}
/**
* When called, indicates that compression should end with the current
* contents of the input buffer.
*/
public void finish() {
synchronized (zsRef) {
finish = true;
}
}
/**
* Returns true if the end of the compressed data output stream has
* been reached.
* #return true if the end of the compressed data output stream has
* been reached
*/
public boolean finished() {
synchronized (zsRef) {
return finished;
}
}
/**
* Fills specified buffer with compressed data. Returns actual number
* of bytes of compressed data. A return value of 0 indicates that
* needsInput() should be called in order to determine if more input
* data is required.
* #param b the buffer for the compressed data
* #param off the start offset of the data
* #param len the maximum number of bytes of compressed data
* #return the actual number of bytes of compressed data
*/
public int deflate(byte[] b, int off, int len) {
if (b == null) {
throw new NullPointerException();
}
if (off < 0 || len < 0 || off > b.length - len) {
throw new ArrayIndexOutOfBoundsException();
}
synchronized (zsRef) {
ensureOpen();
return deflateBytes(zsRef.address(), b, off, len);
}
}
/**
* Fills specified buffer with compressed data. Returns actual number
* of bytes of compressed data. A return value of 0 indicates that
* needsInput() should be called in order to determine if more input
* data is required.
* #param b the buffer for the compressed data
* #return the actual number of bytes of compressed data
*/
public int deflate(byte[] b) {
return deflate(b, 0, b.length);
}
/**
* Returns the ADLER-32 value of the uncompressed data.
* #return the ADLER-32 value of the uncompressed data
*/
public int getAdler() {
synchronized (zsRef) {
ensureOpen();
return getAdler(zsRef.address());
}
}
/**
* Returns the total number of uncompressed bytes input so far.
*
* <p>Since the number of bytes may be greater than
* Integer.MAX_VALUE, the {#link #getBytesRead()} method is now
* the preferred means of obtaining this information.</p>
*
* #return the total number of uncompressed bytes input so far
*/
public int getTotalIn() {
return (int) getBytesRead();
}
/**
* Returns the total number of uncompressed bytes input so far.</p>
*
* #return the total (non-negative) number of uncompressed bytes input so far
* #since 1.5
*/
public long getBytesRead() {
synchronized (zsRef) {
ensureOpen();
return getBytesRead(zsRef.address());
}
}
/**
* Returns the total number of compressed bytes output so far.
*
* <p>Since the number of bytes may be greater than
* Integer.MAX_VALUE, the {#link #getBytesWritten()} method is now
* the preferred means of obtaining this information.</p>
*
* #return the total number of compressed bytes output so far
*/
public int getTotalOut() {
return (int) getBytesWritten();
}
/**
* Returns the total number of compressed bytes output so far.</p>
*
* #return the total (non-negative) number of compressed bytes output so far
* #since 1.5
*/
public long getBytesWritten() {
synchronized (zsRef) {
ensureOpen();
return getBytesWritten(zsRef.address());
}
}
/**
* Resets deflater so that a new set of input data can be processed.
* Keeps current compression level and strategy settings.
*/
public void reset() {
synchronized (zsRef) {
ensureOpen();
reset(zsRef.address());
finish = false;
finished = false;
off = len = 0;
}
}
/**
* Closes the compressor and discards any unprocessed input.
* This method should be called when the compressor is no longer
* being used, but will also be called automatically by the
* finalize() method. Once this method is called, the behavior
* of the Deflater object is undefined.
*/
public void end() {
synchronized (zsRef) {
long addr = zsRef.address();
zsRef.clear();
if (addr != 0) {
end(addr);
buf = null;
}
}
}
/**
* Closes the compressor when garbage is collected.
*/
protected void finalize() {
end();
}
private void ensureOpen() {
assert Thread.holdsLock(zsRef);
if (zsRef.address() == 0)
throw new NullPointerException("Deflater has been closed");
}
private static native void initIDs();
private native static long init(int level, int strategy, boolean nowrap);
private native static void setDictionary(long addr, byte[] b, int off,
int len);
private native int deflateBytes(long addr, byte[] b, int off, int len);
private native static int getAdler(long addr);
private native static long getBytesRead(long addr);
private native static long getBytesWritten(long addr);
private native static void reset(long addr);
private native static void end(long addr);
}
Your problem is not in java code.
There can be the following reasons:
the library that you are using or other library that your library depends on is not in library path (java.library.path option)
One of the native libraries you try to use does not match your platforms. For example you are running on 32 bit platform and trying to use 64bit library.
Take a look here: How to add native library to "java.library.path" with Eclipse launch (instead of overriding it)