Develop Reference

Not able to get 4 leading zeros in sha256 hash proof of work- ever (Java)

I'm build a blockchain app.
When I run tests in main, no matter what I do, no matter how much time I give it, when I log different things out, I'm unable to get 4 leading zeroes and so complete a difficulty level of 4. I see the log of the binary hashes and many times they have repeating elements, 1111 for instance, but never 0000 until my time is hit and the difficulty decreases to three. I have no idea why.
I borrowed the hash algorithm from an online source and I checked its output against an online hasher and it checked out.
I know with each level of difficulty it increases exponentially but 2^4 is still only 16 and I see other repeating numbers (1111, 1010, any combination except 0000). Is there any reason why this might be the case?
I wanted to provide an abundance of code rather than a shortage. Logically it makes no sense why randomly if all numbers were equally possible, it woudln't turn up 0000* (e.g. 0000101011at some point). therefore Four zeros must not be possible, but why? I waited 100 seconds mutliple times and saw other numbers repeat themselves. I saw it hit at exactly 4 or 3 or 2 seconds each time on the dot when difficulty went to three. When I start at difficulty 5 (genesis block) it will never solve- I'm sure even if I left it running overnight. So what could be going on?
package privblock.gerald.ryan;
import java.nio.charset.StandardCharsets;
import java.security.NoSuchAlgorithmException;
import java.time.Instant;
import java.util.Arrays;
import java.util.Date; // gets time in ms.
import privblock.gerald.ryan.util.CryptoHash;
/**
*
* #author Gerald Ryan Block Class of blockchain app
*
* Description: The block hash is the result of the timestamp, the
* last_hash, the data, the difficulty and the nonce
*
*/
public class Block {
long timestamp;
String lastHash;
String hash;
String[] data;
int difficulty;
int nonce;
// Millisecond basis
;
static long MILLISECONDS = 1;
static long SECONDS = 1000 * MILLISECONDS;
static long MINE_RATE = 2 * SECONDS;
/**
* A block is a unit of storage for a blockchain that supports a cryptocurrency.
*
* #param timestamp
* #param lastHash
* #param hash
* #param data
* #param difficulty
* #param nonce
*/
public Block(long timestamp, String lastHash, String hash, String[] data, int difficulty, int nonce) {
super();
this.timestamp = timestamp;
this.lastHash = lastHash;
this.hash = hash;
this.data = data;
this.difficulty = difficulty;
this.nonce = nonce;
}
public String toString() {
return "\n-----------BLOCK--------\ntimestamp: " + this.timestamp + "\nlastHash: " + this.lastHash + "\nhash: "
+ this.hash + "\ndifficulty: " + this.getDifficulty() + "\nNonce: " + this.nonce
+ "\n-----------------------\n";
}
/**
* Mine a block based on given last block and data until a block hash is found
* that meets the leading 0's Proof of Work requirement.
*
* #param last_block
* #param data
* #return
* #throws NoSuchAlgorithmException
*/
public static Block mine_block(Block last_block, String[] data) throws NoSuchAlgorithmException {
long timestamp = new Date().getTime();
String last_hash = last_block.getHash();
int difficulty = Block.adjust_difficulty(last_block, timestamp);
int nonce = 0;
String hash = CryptoHash.getSHA256(timestamp, last_block.getHash(), data, difficulty, nonce);
String proof_of_work = CryptoHash.n_len_string('0', difficulty);
// System.out.println("Proof of work " + proof_of_work);
String binary_hash = CryptoHash.hex_to_binary(hash);
// System.out.println("binary hash " + binary_hash);
String binary_hash_work_end = binary_hash.substring(0, difficulty);
// System.out.println("binary_Hash_work_end " + binary_hash_work_end);
System.out.println("Difficulty: " + difficulty);
while (!proof_of_work.equalsIgnoreCase(binary_hash_work_end)) {
// System.out.println("Working");
nonce += 1;
timestamp = new Date().getTime();
difficulty = Block.adjust_difficulty(last_block, timestamp);
hash = CryptoHash.getSHA256(timestamp, last_block.getHash(), data, difficulty, nonce);
proof_of_work = CryptoHash.n_len_string('0', difficulty);
binary_hash = CryptoHash.hex_to_binary(hash);
binary_hash_work_end = binary_hash.substring(0, difficulty);
// System.out.println(binary_hash_work_end);
// System.out.println(binary_hash);
// System.out.println(proof_of_work);
}
System.out.println("Solved at Difficulty: " + difficulty);
// System.out.println("Proof of work requirement " + proof_of_work);
// System.out.println("binary_Hash_work_end " + binary_hash_work_end);
// System.out.println("binary hash " + binary_hash);
System.out.println("BLOCK MINED");
return new Block(timestamp, last_hash, hash, data, difficulty, nonce);
}
/**
* Generate Genesis block
*
* #return
*/
public static Block genesis_block() {
long timestamp = 1;
String last_hash = "genesis_last_hash";
String hash = "genesis_hash";
String[] data = { "buy", "privcoin" };
int difficulty = 4;
int nonce = 0;
return new Block(timestamp, last_hash, hash, data, difficulty, nonce);
}
/**
* Calculate the adjusted difficulty according to the MINE_RATE. Increase the
* difficulty for quickly mined blocks. Decrease the difficulty for slowly mined
* blocks.
*
* #param last_block
* #param new_timestamp
*/
public static int adjust_difficulty(Block last_block, long new_timestamp) {
long time_diff = new_timestamp - last_block.getTimestamp();
// System.out.println(time_diff);
if (time_diff < MINE_RATE) {
// System.out.println("Increasing difficulty");
return last_block.getDifficulty() + 1;
} else if (last_block.getDifficulty() - 1 > 0) {
// System.out.println("Decreasing difficulty");
return last_block.getDifficulty() - 1;
} else {
return 1;
}
}
/**
* Validate block by enforcing following rules: - Block must have the proper
* last_hash reference - Block must meet the proof of work requirements -
* difficulty must only adjust by one - block hash must be a valid combination
* of block fields
*
* #param last_block
* #param block
* #return
* #throws NoSuchAlgorithmException
*/
public static boolean is_valid_block(Block last_block, Block block) throws NoSuchAlgorithmException {
String binary_hash = CryptoHash.hex_to_binary(block.getHash());
char[] pow_array = CryptoHash.n_len_array('0', block.getDifficulty());
char[] binary_char_array = CryptoHash.string_to_charray(binary_hash);
if (!block.getLastHash().equalsIgnoreCase(last_block.getHash())) {
System.out.println("The last hash must be correct");
return false;
// Throw exception the last hash must be correct
}
if (!Arrays.equals(pow_array, Arrays.copyOfRange(binary_char_array, 0, block.getDifficulty()))) {
System.out.println("Proof of work requirement not met");
return false;
// throw exception - proof of work requirement not met
}
if (Math.abs(last_block.difficulty - block.difficulty) > 1) {
System.out.println("Block difficulty must adjust by one");
return false;
// throw exception: The block difficulty must only adjust by 1
}
String reconstructed_hash = CryptoHash.getSHA256(block.getTimestamp(), block.getLastHash(), block.getData(),
block.getDifficulty(), block.getNonce());
if (!block.getHash().equalsIgnoreCase(reconstructed_hash)) {
System.out.println("The block hash must be correct");
System.out.println(block.getHash());
System.out.println(reconstructed_hash);
return false;
// throw exception: the block hash must be correct
}
System.out.println("You have mined a valid block");
return true;
}
public int getDifficulty() {
return difficulty;
}
public long getTimestamp() {
return timestamp;
}
public String getHash() {
return hash;
}
public String getLastHash() {
return lastHash;
}
public String[] getData() {
return data;
}
public int getNonce() {
return nonce;
}
public static void main(String[] args) throws NoSuchAlgorithmException {
// String md = CryptoHash.getSHA256("foobar");
Block genesis = genesis_block();
System.out.println(genesis.toString());
// Block bad_block = Block.mine_block(genesis, new String[] { "watch", "AOT" });
// bad_block.lastHash = "evil data";
// System.out.println(bad_block.toString());
Block good_block = mine_block(genesis, new String[] { "foo", "bar" });
System.out.println(good_block.toString());
// System.out.println(mine_block(new_block, new String[] { "crypto", "is", "fun" }).toString());
// System.out.println(Block.is_valid_block(genesis, bad_block)); // returns false as expected
System.out.println(Block.is_valid_block(genesis, good_block));
System.out.println(CryptoHash.hex_to_binary(good_block.getHash()));
Block good_block2 = mine_block(good_block, new String[] { "bar", "foo" });
Block good_block3 = mine_block(good_block2, new String[] { "bar", "foo" });
Block good_block4 = mine_block(good_block3, new String[] { "bar", "foo" });
// Block good_block5 = mine_block(good_block4, new String[] {"bar", "foo"});
// Block good_block6 = mine_block(good_block5, new String[] {"bar", "foo"});
}
}
package privblock.gerald.ryan.util;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.HashMap;
public class CryptoHash {
static HashMap<Character, String> HEX_TO_BIN_TABLE;
static {
HEX_TO_BIN_TABLE = new HashMap<Character, String>();
HEX_TO_BIN_TABLE.put('0', "0000");
HEX_TO_BIN_TABLE.put('1', "0001");
HEX_TO_BIN_TABLE.put('2', "0010");
HEX_TO_BIN_TABLE.put('3', "0011");
HEX_TO_BIN_TABLE.put('4', "0100");
HEX_TO_BIN_TABLE.put('5', "0101");
HEX_TO_BIN_TABLE.put('6', "0110");
HEX_TO_BIN_TABLE.put('7', "0111");
HEX_TO_BIN_TABLE.put('8', "1000");
HEX_TO_BIN_TABLE.put('9', "1001");
HEX_TO_BIN_TABLE.put('a', "1010");
HEX_TO_BIN_TABLE.put('b', "1011");
HEX_TO_BIN_TABLE.put('c', "1100");
HEX_TO_BIN_TABLE.put('d', "1101");
HEX_TO_BIN_TABLE.put('e', "1110");
HEX_TO_BIN_TABLE.put('f', "1111");
}
public static String getSHA256(String... sarray) throws NoSuchAlgorithmException {
String s = concat(sarray);
// System.out.printf("Hashing \"%s\"\n", s);
MessageDigest md;
md = MessageDigest.getInstance("SHA-256");
byte[] b = md.digest(s.getBytes(StandardCharsets.UTF_8));
BigInteger number = new BigInteger(1, b);
StringBuilder hexString = new StringBuilder(number.toString(16));
while (hexString.length() < 32) {
hexString.insert(0, '0');
}
String mds = hexString.toString();
// System.out.printf("hash is:\n%s\n", mds);
return hexString.toString();
}
public static String getSHA256(long timestamp, String last_hash, String[] data, int difficulty, int nonce)
throws NoSuchAlgorithmException {
String s = "";
s += Long.toString(timestamp);
s += last_hash;
s += concat(data);
s += Integer.toString(difficulty);
s += Integer.toString(nonce);
// System.out.printf("Hashing \"%s\"\n", s);
MessageDigest md;
md = MessageDigest.getInstance("SHA-256");
byte[] b = md.digest(s.getBytes(StandardCharsets.UTF_8));
BigInteger number = new BigInteger(1, b);
StringBuilder hexString = new StringBuilder(number.toString(16));
// System.out.println(hexString);
while (hexString.length() < 32) {
hexString.insert(0, '0');
}
String messageDigestString = hexString.toString();
// System.out.printf("hash is:\n%s\n", messageDigestString);
return hexString.toString();
}
public static char[] n_len_array(char c, int n) {
char[] ch = new char[n];
for (int i = 0; i<n; i++) {
ch[i] = c;
}
return ch;
}
public static String n_len_string(char c, int n) {
String s = "";
for (int i = 0; i<n; i++) {
s += c;
}
return s;
}
public static String concat(String... args) {
String s = "";
for (String $ : args) {
s += $;
}
// System.out.println(s);
return s;
}
public static char[] string_to_charray(String str) {
char[] ch = new char[str.length()];
for (int i = 0; i < str.length(); i++) {
ch[i] = str.charAt(i);
}
return ch;
}
public static String string_to_hex(String arg) {
return String.format("%064x", new BigInteger(1, arg.getBytes(StandardCharsets.UTF_8)));
}
public static String hex_to_binary(String hex_string) {
String binary_string = "";
for (int i = 0; i < hex_string.length(); i++) {
binary_string += HEX_TO_BIN_TABLE.get(hex_string.charAt(i));
}
return binary_string;
}
public static String string_to_binary(String raw_string) {
String hex_string = string_to_hex(raw_string);
String bin_string = hex_to_binary(hex_string);
return bin_string;
}
}
ps here's an example of a log I created. I created other cleaner logs too but this shows what we're working with. The first item represents time in milliseconds. The second represents the first four digits of the hash, which is directly below it, followed by the level of difficulty requirement string (what the second item needs to be, length n = difficulty level). The hash just never leads with four zeros, ever, so my hash function or call to the function must be broken in some way.
6479
1000
1000001010111011100110111010100100111010101001111110010101011101101101110000110100110110110000001010001000000010110001100111100111010100110001001001110111011010011100110000011111110100000100000100000010100001000110000111000101100010001111011000110011111101
0000
6479
0101
0101110111010100101010100000001011100011000001110001011011001101001111101011010011000111101101111111001001001010100110101101100111111011001011100101111000011100010001000000000011000111010000101101001000001010101010111001010000101001110011111101011011011000
0000
6479
1000
1000000001000101001110001110110000110111001101100001011000111010111110001011011010011111111101011001110011001001111011011110110010101010101100011011001001110001100010010101001011100001101011011101010000000100111100011011110100000101100111010100100110011101
0000
6479

I figured out the problem. It is indeed often returning 4 leading zeroes but the code as structured is clipping them off (because it doesn't think they have meaning). I noticed by logging that the length is not always a fixed 64byte/256 bit string. Here's the output:
256
1101111000010000100001110001010001010000001010111001100011010011110010001001010001010010100110111000110010000010001110110100100101000000001111111110011100000001010100000111001000111101010001010100110100000000111000100001000000010010010111011110110011110111
256
011001111101001000011111011001111110010110000011001011111010001011010110010100001011010011010010111101100010010111000010110010110111110001010101100000000101001000111110100111011100001110010010101011011000000101100001101110101101010001110000111111110000
252
0001100101110011101000000011000101011100111101110100111110100101110110011100010110001011000110010011110110011001100111010001100100011001011000001011100011011011011011101000111000011100100011011011011000101010011101000110101011000110011100111010000011000011
256
1100110001001001110001100111100010101100100010110111100111001010011011111111100010100110110000010000101000010111111010010101110001100010101010111111111111001011010111010100001010000010111100100100111000010101011000110000100000100111010001000011000000010000
256
So that's solved, or at least I understand the problem. It's amazing what sleep will do.

Check if an IP is contained in a set of ranges?

Does anyone know of a clean way to check if an IP is contained in a set of ranges?
I'm getting a response that looks like this:
"ipRange": "10.12.0.0/16,10.80.0.0/15,10.83.0.0/16,10.101.0.0-10.103.255.255,10.108.0.0/16,10.121.0.0/16,10.123.0.0/16,10.127.0.0/16,10.129.0.0/16,10.131.0.0/16,10.133.0.0/16,10.135.0.0-10.139.255.255,10.208.0.0/14,10.215.0.0/16,10.218.0.0/15,10.233.0.0/16,172.17.128.0-172.19.255.255,172.24.0.0/15,192.168.0.0/16"
And I need to see if an IP is in any of those works. The ones with "slash" notation should be straight forward, but what about the ranges where they have "-"?

An IPv4 address is nothing else than a unsigned int32. The easy way to check whether IP address A with network mask X is in subnet B with network mask Y:
convert A to uint32
convert B to uint32
convert Y to uint32
if (A & Y) == (B & Y) then yes

I've commented the code below. Basically, I split the "ipRanges" into distinct ipAddresses or ipAddressRanges by looking for ",". Then test the given ip to each of those to see if it is good.
String ipRange = "10.12.0.0/16,10.80.0.0/15,10.83.0.0/16,10.101.0.0-10.103.255.255,10.108.0.0/16,10.121.0.0/16,10.123.0.0/16,10.127.0.0/16,10.129.0.0/16,10.131.0.0/16,10.133.0.0/16,10.135.0.0-10.139.255.255,10.208.0.0/14,10.215.0.0/16,10.218.0.0/15,10.233.0.0/16,172.17.128.0-172.19.255.255,172.24.0.0/15,192.168.0.0/16";
public IPAddressTest() {
String ipToTest = "10.101.0.0";
System.out.println(String.format("[%s] is in range (%b)", ipToTest, isInRange(ipToTest)));
}
private Boolean isInRange(String ipToTest) {
String[] ranges = ipRange.split(",");
for (String _range : ranges) {
String range = _range.indexOf("/") > -1 ? _range.substring(0, _range.indexOf("/")) : _range; // cut off the /xx if it exists
boolean test = test(ipToTest, range);
if (test) {
return true;
}
}
return false;
}
private Boolean test(String ipToTest, String ipRange) {
String[] ranges = ipRange.split("-"); // Do we have a range or just a single ip
String loRange = ranges[0]; // grab the first
String hiRange = ranges.length > 1 ? ranges[1] : null; // If there a range grab the top end, else nothing
if (hiRange == null) { // if a single ip, just do a string compare
return ipToTest.equals(loRange);
} else { // if a range then make sure each part is between the min and max
String[] partsToTest = ipToTest.split("\\."); // get each part of the ip
String[] partsLoRange = loRange.split("\\."); // get each part of the ip
String[] partsHiRange = hiRange.split("\\."); // get each part of the ip
int test = 0; // this will be our litmus test. if it = 15 when we are done, then each part is good, 63 of ipv6
for (int i = 0; i < 4; i++) { // change 4 to 6 for ipv6
int value = Integer.parseInt(partsToTest[i]); //get the int
int lo = Integer.parseInt(partsLoRange[i]); //get the int
int hi = Integer.parseInt(partsHiRange[i]); //get the int
if (value >= lo && value <= hi) { // if we have a good value, set the bit on 'test'
test += Math.pow(2, i);
}
}
if (test == 15) {
return true;
}
}
return false;
}
public static void main(String[] args) {
new IPAddressTest();
}

converting binary/decimal fraction to hexadecimal in java -unexpected output

I am trying to write a program that converts binary(with or without fraction) inputs into hex which is nearly done but unfortunately in the hex output the point (".")is missing.
Suppose my expected output is e7.6 , but i am getting e76 instead.
only the "." is missing.
here is my BinToHex class..
import java.io.*;
//tried to convert the binary into dec and then dec to hex
public class BinToHex {
double tempDec,fractionpart;
long longofintpart,templongDec;
String inpu ="11100111.011";
String hexOutput=null,tempDecString,hex = null;
static int i = 1;
public void convertbintohex() {
if (inpu.contains(".")) {
int placesAfterPoint = inpu.length() - inpu.indexOf(".") - 1;//every thing
long numerator = Long.parseLong(inpu.replace(".", ""), 2);//goes
double decimalOfInput = ((double) numerator) / (1L << placesAfterPoint);//alright till here
while (true) {
tempDec = decimalOfInput * 16;
if (tempDec == (int)tempDec) {
tempDecString = String.valueOf((long)tempDec);
templongDec = Long.parseLong(tempDecString, 10);
hexOutput = Long.toHexString(templongDec);
break;
} else {
longofintpart = (long)tempDec;
hex=Long.toHexString(longofintpart);
if(i==1){
hexOutput = hex + ".";
i=i+1;
}else{
hexOutput = hexOutput + hex;
}
fractionpart = tempDec-(int)tempDec;
decimalOfInput = fractionpart;
}
}
} else {
// this part is ok
tempDecString = String.valueOf(Integer.parseInt(inpu, 2));
templongDec = Long.parseLong(tempDecString, 10);
hexOutput = Long.toHexString(templongDec);
}
System.out.println(hexOutput);
}
}
my main Test class..
public class Test{
public static void main(String args[]){
BinToHex i = new BinToHex();
i.convertbintohex();
}
}
I am stuck!
plz help .

really, not i can't resist to write a solution... after having such long comments, it jst takes me some minutes ^^
final int CODEBASE = 16;
String input = "11100111.011";
//lets see if we have a '.' in our String
if (input.indexOf(".") > 0) {
//yes, we have one - so we can split the string by '.'
String splits = input.split(".");
//the part left of the dot
String beforeDot = splits[0];
//the part right of the dot
String afterDot = splits[1];
//it's a incomplete input, we must fill up with
//trailing zeros according to out code base
afterDot.fillTrailingZeros(afterDot, CODEBASE);
//now we can parse the input
int asIntBefore = Integer.parseInt(beforeDots, 2);
int asIntAfter = Integer.parseInt(afterDot , 2);
} else {
//use your working code for
//input wthoput dot HERE
}
//fills trailing zeros to input String
String fillTrailingZeros(String input, int base){
//as long as our String is shorter than the codebase...
while (input.length() < base){
//...we have to add trailing zeros
input = input +"0";
}
return input;
}

At last found a proper algorithm for converting decimal(with or without fraction) to hex.
besides, binary(with or without fraction) to decimal in Java is here
The algorithm for converting decimal(with or without fraction) into hex in Java
import java.math.*;
public class DecimalToHex{
public String decimalToHex(String decInpString){
StringBuilder hexOut = new StringBuilder();
double doubleOfDecInp = Double.parseDouble(decInpString);
if(doubleOfDecInp < 0){
hexOut = hexOut.append("-");
doubleOfDecInp = -doubleOfDecInp;
}
BigInteger beforedot = new BigDecimal(doubleOfDecInp).toBigInteger();
hexOut.append(beforedot.toString(16));
BigDecimal bfd =new BigDecimal(beforedot);
doubleOfDecInp = doubleOfDecInp - bfd.doubleValue();
if(doubleOfDecInp == 0){
return hexOut.toString();
}
hexOut.append(".");
for (int i = 0; i < 16; ++i) {
doubleOfDecInp = doubleOfDecInp * 16;
int digit = (int)doubleOfDecInp;
hexOut.append(Integer.toHexString(digit));
doubleOfDecInp = doubleOfDecInp - digit;
if (doubleOfDecInp == 0)
break;
}
return hexOut.toString();
}
public static void main(String args[]){
String decimalInp = "-0.767";
String out ;
DecimalToHex i = new DecimalToHex();
out = i.decimalToHex(decimalInp);
System.out.println(out);
}
}

In Java, when to write separate methods for different jobs - best practices?

I wrote a small program that validates IP addresses, esp. strips off some metadata like range and network mask of an IP v4 address before passig it to the IPAddress Checker.
While stripping off the network mask of an IP address I need the resulting IP address (I call it the base IP address) and the network mask in separate variables, for further validation.
I was wondering if the way I did it in the cutOffNetMask method is ok or if I should split the jobs
extract base ip
extract the network mask
into separate methods?
Is the way I set the private Integer netMask; ìn the cutOffNetMask method the state of the art or how does the elegant way doing it looks like?
Here is the snippet:
public class IPAddress {
private String base;
private Integer netMask;
private boolean ipRangeValid;
private boolean netMaskValid;
public IPAddress(String value){
ipAddress = value;
base = ipAddress;
base.replace(" ","");
if(isIPv6Address(base) == false){
if(hasNetMask(base) == true){
base = cutOffNetMask(base);
}
if(hasIPRange(base) == true){
base = stripOffIPRange(base);
ipRangeValid = ipRangeValid(octet1, octet2, octet3, octet4);
} else {
ipRangeValid = true;
}
}
}
private static boolean hasNetMask(String base){
return (base.indexOf("/") != -1);
}
private static String cutOffNetMask(String base){
netMask = base.substring(base.indexOf("/") + 1);
return base.replace(netMask, "");
}
private String stripOffIPRange(String base){
String[] octets = base.split("\\.");
base = "";
octet1 = new IPOctet(octets[0]);
base = base + (octet1.getBase() + ".");
octet2 = new IPOctet(octets[1]);
base = base + (octet2.getBase() + ".");
octet3 = new IPOctet(octets[2]);
base = base + (octet3.getBase() + ".");
octet4 = new IPOctet(octets[3]);
base = base + (octet4.getBase());
return base;
}

Binary search in a sorted (memory-mapped ?) file in Java

I am struggling to port a Perl program to Java, and learning Java as I go. A central component of the original program is a Perl module that does string prefix lookups in a +500 GB sorted text file using binary search
(essentially, "seek" to a byte offset in the middle of the file, backtrack to nearest newline, compare line prefix with the search string, "seek" to half/double that byte offset, repeat until found...)
I have experimented with several database solutions but found that nothing beats this in sheer lookup speed with data sets of this size. Do you know of any existing Java library that implements such functionality? Failing that, could you point me to some idiomatic example code that does random access reads in text files?
Alternatively, I am not familiar with the new (?) Java I/O libraries but would it be an option to memory-map the 500 GB text file (I'm on a 64-bit machine with memory to spare) and do binary search on the memory-mapped byte array? I would be very interested to hear any experiences you have to share about this and similar problems.

I am a big fan of Java's MappedByteBuffers for situations like this. It is blazing fast. Below is a snippet I put together for you that maps a buffer to the file, seeks to the middle, and then searches backwards to a newline character. This should be enough to get you going?
I have similar code (seek, read, repeat until done) in my own application, benchmarked
java.io streams against MappedByteBuffer in a production environment and posted the results on my blog (Geekomatic posts tagged 'java.nio' ) with raw data, graphs and all.
Two second summary? My MappedByteBuffer-based implementation was about 275% faster. YMMV.
To work for files larger than ~2GB, which is a problem because of the cast and .position(int pos), I've crafted paging algorithm backed by an array of MappedByteBuffers. You'll need to be working on a 64-bit system for this to work with files larger than 2-4GB because MBB's use the OS's virtual memory system to work their magic.
public class StusMagicLargeFileReader {
private static final long PAGE_SIZE = Integer.MAX_VALUE;
private List<MappedByteBuffer> buffers = new ArrayList<MappedByteBuffer>();
private final byte raw[] = new byte[1];
public static void main(String[] args) throws IOException {
File file = new File("/Users/stu/test.txt");
FileChannel fc = (new FileInputStream(file)).getChannel();
StusMagicLargeFileReader buffer = new StusMagicLargeFileReader(fc);
long position = file.length() / 2;
String candidate = buffer.getString(position--);
while (position >=0 && !candidate.equals('\n'))
candidate = buffer.getString(position--);
//have newline position or start of file...do other stuff
}
StusMagicLargeFileReader(FileChannel channel) throws IOException {
long start = 0, length = 0;
for (long index = 0; start + length < channel.size(); index++) {
if ((channel.size() / PAGE_SIZE) == index)
length = (channel.size() - index * PAGE_SIZE) ;
else
length = PAGE_SIZE;
start = index * PAGE_SIZE;
buffers.add(index, channel.map(READ_ONLY, start, length));
}
}
public String getString(long bytePosition) {
int page = (int) (bytePosition / PAGE_SIZE);
int index = (int) (bytePosition % PAGE_SIZE);
raw[0] = buffers.get(page).get(index);
return new String(raw);
}
}

I have the same problem. I am trying to find all lines that start with some prefix in a sorted file.
Here is a method I cooked up which is largely a port of Python code found here: http://www.logarithmic.net/pfh/blog/01186620415
I have tested it but not thoroughly just yet. It does not use memory mapping, though.
public static List<String> binarySearch(String filename, String string) {
List<String> result = new ArrayList<String>();
try {
File file = new File(filename);
RandomAccessFile raf = new RandomAccessFile(file, "r");
long low = 0;
long high = file.length();
long p = -1;
while (low < high) {
long mid = (low + high) / 2;
p = mid;
while (p >= 0) {
raf.seek(p);
char c = (char) raf.readByte();
//System.out.println(p + "\t" + c);
if (c == '\n')
break;
p--;
}
if (p < 0)
raf.seek(0);
String line = raf.readLine();
//System.out.println("-- " + mid + " " + line);
if (line.compareTo(string) < 0)
low = mid + 1;
else
high = mid;
}
p = low;
while (p >= 0) {
raf.seek(p);
if (((char) raf.readByte()) == '\n')
break;
p--;
}
if (p < 0)
raf.seek(0);
while (true) {
String line = raf.readLine();
if (line == null || !line.startsWith(string))
break;
result.add(line);
}
raf.close();
} catch (IOException e) {
System.out.println("IOException:");
e.printStackTrace();
}
return result;
}

I am not aware of any library that has that functionality. However, a correct code for a external binary search in Java should be similar to this:
class ExternalBinarySearch {
final RandomAccessFile file;
final Comparator<String> test; // tests the element given as search parameter with the line. Insert a PrefixComparator here
public ExternalBinarySearch(File f, Comparator<String> test) throws FileNotFoundException {
this.file = new RandomAccessFile(f, "r");
this.test = test;
}
public String search(String element) throws IOException {
long l = file.length();
return search(element, -1, l-1);
}
/**
* Searches the given element in the range [low,high]. The low value of -1 is a special case to denote the beginning of a file.
* In contrast to every other line, a line at the beginning of a file doesn't need a \n directly before the line
*/
private String search(String element, long low, long high) throws IOException {
if(high - low < 1024) {
// search directly
long p = low;
while(p < high) {
String line = nextLine(p);
int r = test.compare(line,element);
if(r > 0) {
return null;
} else if (r < 0) {
p += line.length();
} else {
return line;
}
}
return null;
} else {
long m = low + ((high - low) / 2);
String line = nextLine(m);
int r = test.compare(line, element);
if(r > 0) {
return search(element, low, m);
} else if (r < 0) {
return search(element, m, high);
} else {
return line;
}
}
}
private String nextLine(long low) throws IOException {
if(low == -1) { // Beginning of file
file.seek(0);
} else {
file.seek(low);
}
int bufferLength = 65 * 1024;
byte[] buffer = new byte[bufferLength];
int r = file.read(buffer);
int lineBeginIndex = -1;
// search beginning of line
if(low == -1) { //beginning of file
lineBeginIndex = 0;
} else {
//normal mode
for(int i = 0; i < 1024; i++) {
if(buffer[i] == '\n') {
lineBeginIndex = i + 1;
break;
}
}
}
if(lineBeginIndex == -1) {
// no line begins within next 1024 bytes
return null;
}
int start = lineBeginIndex;
for(int i = start; i < r; i++) {
if(buffer[i] == '\n') {
// Found end of line
return new String(buffer, lineBeginIndex, i - lineBeginIndex + 1);
return line.toString();
}
}
throw new IllegalArgumentException("Line to long");
}
}
Please note: I made up this code ad-hoc: Corner cases are not tested nearly good enough, the code assumes that no single line is larger than 64K, etc.
I also think that building an index of the offsets where lines start might be a good idea. For a 500 GB file, that index should be stored in an index file. You should gain a not-so-small constant factor with that index because than there is no need to search for the next line in each step.
I know that was not the question, but building a prefix tree data structure like (Patrica) Tries (on disk/SSD) might be a good idea to do the prefix search.

This is a simple example of what you want to achieve. I would probably first index the file, keeping track of the file position for each string. I'm assuming the strings are separated by newlines (or carriage returns):
RandomAccessFile file = new RandomAccessFile("filename.txt", "r");
List<Long> indexList = new ArrayList();
long pos = 0;
while (file.readLine() != null)
{
Long linePos = new Long(pos);
indexList.add(linePos);
pos = file.getFilePointer();
}
int indexSize = indexList.size();
Long[] indexArray = new Long[indexSize];
indexList.toArray(indexArray);
The last step is to convert to an array for a slight speed improvement when doing lots of lookups. I would probably convert the Long[] to a long[] also, but I did not show that above. Finally the code to read the string from a given indexed position:
int i; // Initialize this appropriately for your algorithm.
file.seek(indexArray[i]);
String line = file.readLine();
// At this point, line contains the string #i.

If you are dealing with a 500GB file, then you might want to use a faster lookup method than binary search - namely a radix sort which is essentially a variant of hashing. The best method for doing this really depends on your data distributions and types of lookup, but if you are looking for string prefixes there should be a good way to do this.
I posted an example of a radix sort solution for integers, but you can use the same idea - basically to cut down the sort time by dividing the data into buckets, then using O(1) lookup to retrieve the bucket of data that is relevant.
Option Strict On
Option Explicit On
Module Module1
Private Const MAX_SIZE As Integer = 100000
Private m_input(MAX_SIZE) As Integer
Private m_table(MAX_SIZE) As List(Of Integer)
Private m_randomGen As New Random()
Private m_operations As Integer = 0
Private Sub generateData()
' fill with random numbers between 0 and MAX_SIZE - 1
For i = 0 To MAX_SIZE - 1
m_input(i) = m_randomGen.Next(0, MAX_SIZE - 1)
Next
End Sub
Private Sub sortData()
For i As Integer = 0 To MAX_SIZE - 1
Dim x = m_input(i)
If m_table(x) Is Nothing Then
m_table(x) = New List(Of Integer)
End If
m_table(x).Add(x)
' clearly this is simply going to be MAX_SIZE -1
m_operations = m_operations + 1
Next
End Sub
Private Sub printData(ByVal start As Integer, ByVal finish As Integer)
If start < 0 Or start > MAX_SIZE - 1 Then
Throw New Exception("printData - start out of range")
End If
If finish < 0 Or finish > MAX_SIZE - 1 Then
Throw New Exception("printData - finish out of range")
End If
For i As Integer = start To finish
If m_table(i) IsNot Nothing Then
For Each x In m_table(i)
Console.WriteLine(x)
Next
End If
Next
End Sub
' run the entire sort, but just print out the first 100 for verification purposes
Private Sub test()
m_operations = 0
generateData()
Console.WriteLine("Time started = " & Now.ToString())
sortData()
Console.WriteLine("Time finished = " & Now.ToString & " Number of operations = " & m_operations.ToString())
' print out a random 100 segment from the sorted array
Dim start As Integer = m_randomGen.Next(0, MAX_SIZE - 101)
printData(start, start + 100)
End Sub
Sub Main()
test()
Console.ReadLine()
End Sub
End Module

I post a gist https://gist.github.com/mikee805/c6c2e6a35032a3ab74f643a1d0f8249c
that is rather complete example based on what I found on stack overflow and some blogs hopefully someone else can use it
import static java.nio.file.Files.isWritable;
import static java.nio.file.StandardOpenOption.READ;
import static org.apache.commons.io.FileUtils.forceMkdir;
import static org.apache.commons.io.IOUtils.closeQuietly;
import static org.apache.commons.lang3.StringUtils.isBlank;
import static org.apache.commons.lang3.StringUtils.trimToNull;
import java.io.File;
import java.io.IOException;
import java.nio.Buffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Path;
public class FileUtils {
private FileUtils() {
}
private static boolean found(final String candidate, final String prefix) {
return isBlank(candidate) || candidate.startsWith(prefix);
}
private static boolean before(final String candidate, final String prefix) {
return prefix.compareTo(candidate.substring(0, prefix.length())) < 0;
}
public static MappedByteBuffer getMappedByteBuffer(final Path path) {
FileChannel fileChannel = null;
try {
fileChannel = FileChannel.open(path, READ);
return fileChannel.map(FileChannel.MapMode.READ_ONLY, 0, fileChannel.size()).load();
}
catch (Exception e) {
throw new RuntimeException(e);
}
finally {
closeQuietly(fileChannel);
}
}
public static String binarySearch(final String prefix, final MappedByteBuffer buffer) {
if (buffer == null) {
return null;
}
try {
long low = 0;
long high = buffer.limit();
while (low < high) {
int mid = (int) ((low + high) / 2);
final String candidate = getLine(mid, buffer);
if (found(candidate, prefix)) {
return trimToNull(candidate);
}
else if (before(candidate, prefix)) {
high = mid;
}
else {
low = mid + 1;
}
}
}
catch (Exception e) {
throw new RuntimeException(e);
}
return null;
}
private static String getLine(int position, final MappedByteBuffer buffer) {
// search backwards to the find the proceeding new line
// then search forwards again until the next new line
// return the string in between
final StringBuilder stringBuilder = new StringBuilder();
// walk it back
char candidate = (char)buffer.get(position);
while (position > 0 && candidate != '\n') {
candidate = (char)buffer.get(--position);
}
// we either are at the beginning of the file or a new line
if (position == 0) {
// we are at the beginning at the first char
candidate = (char)buffer.get(position);
stringBuilder.append(candidate);
}
// there is/are char(s) after new line / first char
if (isInBuffer(buffer, position)) {
//first char after new line
candidate = (char)buffer.get(++position);
stringBuilder.append(candidate);
//walk it forward
while (isInBuffer(buffer, position) && candidate != ('\n')) {
candidate = (char)buffer.get(++position);
stringBuilder.append(candidate);
}
}
return stringBuilder.toString();
}
private static boolean isInBuffer(final Buffer buffer, int position) {
return position + 1 < buffer.limit();
}
public static File getOrCreateDirectory(final String dirName) {
final File directory = new File(dirName);
try {
forceMkdir(directory);
isWritable(directory.toPath());
}
catch (IOException e) {
throw new RuntimeException(e);
}
return directory;
}
}

I had similar problem, so I created (Scala) library from solutions provided in this thread:
https://github.com/avast/BigMap
It contains utility for sorting huge file and binary search in this sorted file...

If you truly want to try memory mapping the file, I found a tutorial on how to use memory mapping in Java nio.