Hello all for the second time,
Initially I was looking for a broad answer, but this thread got blocked for being "too broad"... so I've got no choice, but to go into detail. My apologies if asking the question again is against the forum guidelines, I'm new to stackoverflow so please be kind.
I’ve got data coming into a serial port at 250Hz and I’d like to save it all to a .csv file. Of course draw() is not going to be able to keep up with that rate of data...
At the moment I am using the serialEvent(port) to collect and parse the data. Once parsed out, I'm calling a function in draw to add the data to a new line in a table and then saving that table every 5 seconds...
Yes, I see the obvious flaw that if I'm saving the current data in draw then of course it's not going to be able to save all the data coming in, but rather just the data that happens to be present when the data saving function is called... but I'm not sure of the best way to solve that. A buffer scheme? Or can I have a separate thread that just adds ALL data coming in to a table?
which lead to my initial (broad) question...
Is there a way to save all incoming data to a file without polling?
Thanks to all in advance.. code below:
Twain
import processing.serial.*;
import static javax.swing.JOptionPane.*;
Table table;
String Path = "PathProvidedHere.csv";
String message;
//Some time keeping variables
int hours, minutes, seconds, milliseconds;
float SaveTime;
//Serial port selection
Serial myPort;
String COMx, COMlist = "";
final boolean debug = true;
String portName;
// Data variables
float yaw = 0.0; float pitch = 0.0; float roll = 0.0;
float A1, A2, A3, A4;
float E1, E2, E3, E4;
void setup()
{
//Set up GIU box
size(1024, 768, P3D);
frameRate(250);
smooth();
//Some other setups like fonts, graphs, etc.
//Set up the logging table
table = new Table();
table.addColumn("A1"); table.addColumn("A2"); table.addColumn("A3"); table.addColumn("A4");
table.addColumn(""); table.addColumn("E1"); table.addColumn("E3"); table.addColumn("E4");
table.addColumn(" "); table.addColumn("min"); table.addColumn("sec"); table.addColumn("milli");
portName = chooseCOM();
delay(1000);
}
void draw()
{
SavetoCSV();
//serialEvent(myPort); // read and parse incoming serial message
ACouple();
Unrelated();
FunctionsHere();
if(millis() - SaveTime > 5000)
{
saveTable(table, Path);
SaveTime=millis();
}
}
String chooseCOM()
{
setupP2 = true;
try
{
if (debug) printArray(Serial.list());
int i = Serial.list().length;
if (i != 0)
{
if (i >= 2)
{
// need to check which port the inst uses -
// for now we'll just let the user decide
for (int j = 0; j < i; )
{
COMlist += char(j+'a') + " = " + Serial.list()[j];
if (++j < i) COMlist += ", ";
}
COMx = showInputDialog("Which COM port is correct? (a,b,..):\n"+COMlist);
if (COMx == null) exit();
if (COMx.isEmpty()) exit();
i = int(COMx.toLowerCase().charAt(0) - 'a') + 1;
}
String portName = Serial.list()[i-1];
if (debug) //println(portName + " Selected");
myPort = new Serial(this, portName, 115200); // change baud rate to your liking
myPort.bufferUntil(13); // buffer until CR/LF appears, but not required..
return portName;
}
else
{
showMessageDialog(frame, "Device is not connected to the PC");
exit();
}
}
catch (Exception e)
{ //Print the type of error
showMessageDialog(frame, "COM port is not available (may\nbe in use by another program)");
//println("Error:", e);
exit();
}
return "noPort";
}
void serialEvent(Serial myPort)
{
int newLine = 13; // new line character in ASCII
do
{
message = myPort.readStringUntil(newLine); // read from port until new line
if (message != null)
{
String[] list = split(trim(message), " ");
if (list.length == 4 && list[0].equals("i"))
{
yaw = float(list[1]); // convert to float yaw
pitch = float(list[2]); // convert to float pitch
roll = float(list[3]); // convert to float roll
}
else if (list.length == 5 && list[0].equals("s"))
{
A1 = float(list[1]);
A2 = float(list[2]);
A3 = float(list[3]);
A4 = float(list[4]);
}
else if (list.length >=2 && list[0].equals("b"))
{
Battery = int(list[1]);
}
else if (list.length >= 2 && list[0].equals("m"))
{
MACid = int(list[1]);
}
else
{
//print anything extra to console
//println(message);
}
}
} while (message != null);
}
void SavetoCSV()
{
if (A1 != 0)
{
TableRow newRow = table.addRow();
newRow.setFloat("A1", (A1));
newRow.setFloat("A2", (A2));
newRow.setFloat("A3", (A3));
newRow.setFloat("A4", (A4));
//saveTable(table, Path);
}
}
Additional info:
- Processing P3
- For the record, with the rest of my script I can get draw up to 80hz or so
- I'd be okay with saving all the data and parsing it later
Went the buffer route.... I think I'm getting close now. Unsure if I'm saving the data in the right order or if the saving process will halt the rest of the processes...
Code:
import processing.serial.*;
import static javax.swing.JOptionPane.*;
//Arrays to save the data
LinkedList<Integer> A1c = new LinkedList<Integer>();
LinkedList<Integer> A2c = new LinkedList<Integer>();
LinkedList<Integer> A3c = new LinkedList<Integer>();
LinkedList<Integer> A4c = new LinkedList<Integer>();
int bufferLength = 500;
int bufflen = 0;
//Serial port selection
Serial myPort;
String COMx, COMlist = "";
final boolean debug = true;
String portName;
// Data variables
float yaw = 0.0; float pitch = 0.0; float roll = 0.0;
float A1, A2, A3, A4;
//Data log variables
Table table;
String Path = "PathtoFile.csv";
void setup() {
//Set up GIU box
size(1024, 768, P3D);
frameRate(250);
strokeWeight(50);
smooth();
//Set up the logging table
table = new Table();
table.addColumn("A1"); table.addColumn("A2"); table.addColumn("A3"); table.addColumn("A4");
portName = chooseCOM();
}
void draw() {
//SavetoCSV now called within SerialEvent()
//SavetoCSV();
//serialEvent(myPort); // read and parse incoming serial message
Some();
Unrelated();
FunctionsHere();
}
void serialEvent(Serial myPort) {
int newLine = 13; // new line character in ASCII
do {
message = myPort.readStringUntil(newLine); // read from port until new line
if (message != null) {
String[] list = split(trim(message), " ");
if (list.length == 4 && list[0].equals("i")) {
yaw = float(list[1]); // convert to float yaw
pitch = float(list[2]); // convert to float pitch
roll = float(list[3]); // convert to float roll
} else if (list.length == 5 && list[0].equals("s")) {
A1 = float(list[1]);
A2 = float(list[2]);
A3 = float(list[3]);
A4 = float(list[4]);
if (bufflen < bufferLength) {
A1c.push(int(A1));
A2c.push(int(A2));
A3c.push(int(A3));
A4c.push(int(A4));
bufflen++;
}
else{
bufflen = 0;
SavetoCSV();
}
} else if (list.length >=2 && list[0].equals("b")) {
Battery = int(list[1]);
} else if (list.length >= 2 && list[0].equals("m")) {
MACid = int(list[1]);
} else {
//print anything extra to console
//println(message);
}
}
} while (message != null);
}
void SavetoCSV() {
if (A1 != 0) {
for (int i = bufferLength - 1; i >= 0; i--){
if (i < bufferLength){
TableRow newRow = table.addRow();
newRow.setFloat("A1", (A1c.get(i)));
newRow.setFloat("A2", (A2c.get(i)));
newRow.setFloat("A3", (A3c.get(i)));
newRow.setFloat("A4", (A4c.get(i)));
} else saveTable(table, Path);
}
}
}
String chooseCOM() {
setupP2 = true;
try {
if (debug) printArray(Serial.list());
int i = Serial.list().length;
if (i != 0) {
if (i >= 2) {
// need to check which port the inst uses -
// for now we'll just let the user decide
for (int j = 0; j < i; ) {
COMlist += char(j+'a') + " = " + Serial.list()[j];
if (++j < i) COMlist += ", ";
}
COMx = showInputDialog("Which COM port is correct? (a,b,..):\n"+COMlist);
if (COMx == null) exit();
if (COMx.isEmpty()) exit();
i = int(COMx.toLowerCase().charAt(0) - 'a') + 1;
}
String portName = Serial.list()[i-1];
if (debug) //println(portName + " Selected");
myPort = new Serial(this, portName, 115200); // change baud rate to your liking
myPort.bufferUntil(13); // buffer until CR/LF appears, but not required..
return portName;
} else {
showMessageDialog(frame, "Device is not connected to the PC");
exit();
}
}
catch (Exception e)
{ //Print the type of error
showMessageDialog(frame, "COM port is not available (may\nbe in use by another program)");
//println("Error:", e);
exit();
}
return "noPort";
}
Related
i am experiencing a very strange issue, i wrote code for arduino UNO 3 which is designed to unlock door using servo motor, components attached are following
Servo Motor
1 Red Led (for failure alert)
1 Green Led (for success alert)
1 Buzzer (for audible alert on unlock)
The code is following
#include <Arduino_JSON.h>
#include <SoftwareSerial.h>
#include <EEPROM.h>
#include <Servo.h>
String com = "";
const int buzzer = 6;
const int ledfalse = 8;
const int ledtrue = 13;
const int servo = 11;
Servo myservo;
SoftwareSerial mySerial(2, 3);
void ResetAdmin()
{
for (int i = 0 ; i < EEPROM.length() ; i++)
EEPROM.write(i, 0);
Blink(ledtrue, 2);
}
void WriteAdmin(String admin)
{
byte len = admin.length();
EEPROM.write(0, len);
for (int i = 0; i < len; i++)
{
EEPROM.write(i + 1, admin[i]);
}
Blink(ledtrue, 2);
}
String ReadAdmin()
{
int newStrLen = EEPROM.read(0);
char data[newStrLen + 1];
for (int i = 0; i < newStrLen; i++)
{
data[i] = EEPROM.read(i + 1);
}
data[newStrLen] = '\0';
return String(data);
}
void Unlock()
{
Alert();
myservo.write(0);
delay(500);
myservo.write(90);
delay(6500);
myservo.write(360);
delay(500);
myservo.write(90);
}
void Blink(int type, int times)
{
for(int i = 1; i <= times; i++)
{
digitalWrite(type, HIGH);
delay(80);
digitalWrite(type, LOW);
delay(80);
}
}
void Alert()
{
for(int i = 1; i <= 4; i++)
{
tone(buzzer, 1000);
delay(80);
noTone(buzzer);
delay(80);
}
}
void ProcessCommand(String command)
{
if(command == "unlock")
Unlock(); //not works here
else if(command == "reset")
ResetAdmin();
else
{
Blink(ledfalse, 2);
}
}
void setup()
{
myservo.attach(servo);
mySerial.begin(9600);
pinMode(buzzer, OUTPUT);
pinMode(ledfalse, OUTPUT);
pinMode(ledtrue, OUTPUT);
//Unlock() or Blink(ledtrue, 4) or Alert() works here
digitalWrite(ledtrue, HIGH);
digitalWrite(ledfalse, HIGH);
delay(3000);
digitalWrite(ledtrue, LOW);
digitalWrite(ledfalse, LOW);
}
void loop()
{
while(mySerial.available() > 0)
{
delay(10);
com += (char)Serial.read();
}
if(com.length() > 0)
{
JSONVar doc = JSON.parse(com);
if (JSON.typeof(doc) != "undefined")
{
String admin = ReadAdmin();
if(admin == "")
{
admin = doc["admin"];
WriteAdmin(admin);
}
if(admin == doc["admin"])
{
ProcessCommand((const char*) doc["command"]);
}
else
{
Blink(ledfalse, 2);
}
}
else
{
Blink(ledfalse, 2);
}
com = "";
}
delay(10);
}
The java snippet for sending command is following
private void Unlock() {
if (btSocket != null) {
try {
String payload = "{\"admin\": \"" + getUUID() + "\", \"command\": \"unlock\"}";
btSocket.getOutputStream().write(payload.getBytes());
} catch (Exception e) {
e.printStackTrace();
biometricLoginButton.setImageResource(R.drawable.warning);
failed = true;
Toast.makeText(getBaseContext(), "Error occurred while unlocking", Toast.LENGTH_SHORT).show();
}
} else {
Toast.makeText(getBaseContext(), "Door Lock not connected", Toast.LENGTH_SHORT).show();
biometricLoginButton.setImageResource(R.drawable.warning);
failed = true;
}
}
The issue is when i put the Alert(), Unlock() & Blink() function in the Arduino setup function then alert is working fine and so do the other two, but when the same functions called using Bluetooth signal none of them works. Note that the function is called as servo try to move but not correctly, this shows that Bluetooth is receiving data correctly from android and conditions are being evaluated correctly.
Finally i myself figured out the issue, basically the problem was with the EEPROM of the Arduino the problematic section of the code is following
String admin = ReadAdmin();
if(admin == "")
{
//on first run it never returned empty string, may be my reading function bug
admin = doc["admin"];
WriteAdmin(admin);
}
if(admin == doc["admin"])
{
//due to garbage value compared with actual one this block never executed
rocessCommand((const char*) doc["command"]);
}
else
{
Blink(ledfalse, 2);
}
First condition is for the first run so that when there is no admin stored in the EEPROM, then store the one coming in the JSON, but i don't know why but it was always some garbage value instead of empty or null string that's why the admin was not being matched e.g. the garbage value and the one received in JSON and thus not running the Unlock sequence
i need help. can somebody tell me about concept and example code i need use? the case is, I want input game history using java into mysql (phpmyadmin).
I'm already create like this if S0: 1.0; and all value zero
try
{
connection = dbManager.getConnection();
String bs = "S0:"+s[0]+"; S1:"+s[1]+"; S2:"+s[2]+"; S3:"+s[3]+"; S4:"+s[4]+"; S5:"+s[5]+"; S6:"+s[6]+"; S7:"+s[7]+"; S8:"+s[8]+"; S9:"+s[9]+"; S10:"+s[10]+"; S11:"+s[11]+"; S12:"+s[12]+"; S13:"+s[13]+"; S14:"+s[14]+"; S15:"+s[15]+"; S16:"+s[16]+"; S17:"+s[17]+"; S18:"+s[18]+"; S19:"+s[19]+"; S20:"+s[20]+"; S21:"+s[21]+"; S22:"+s[22]+"; S23:"+s[23]+"; S24:"+s[24]+"; S25:"+s[25]+"; S26:"+s[26]+"; S27:"+s[27]+"; S28:"+s[28]+"; S29:"+s[29]+"; S30:"+s[30]+"; S31:"+s[31]+"; S32:"+s[32]+"; S33:"+s[33]+"; S34:"+s[34]+"; S35:"+s[35]+"; S36:"+s[36]+"";
}
catch (SQLException e)
{
trace("player 1 update error");
}
and it's working and result like this
S0:1.0; S1:0.0; S2:0.0; S3:0.0; S4:0.0; S5:0.0; S6:0.0; S7:0.0; S8:0.0; S9:0.0; S10:0.0; S11:0.0; S12:0.0; S13:0.0; S14:0.0; S15:0.0; S16:0.0; S17:0.0; S18:0.0; S19:0.0; S20:0.0; S21:0.0; S22:0.0; S23:0.0; S24:0.0; S25:0.0; S26:0.0; S27:0.0; S28:0.0; S29:0.0; S30:0.0; S31:0.0; S32:0.0; S33:0.0; S34:0.0; S35:0.0; S36:0.0
but it's not efficient, the question is how i can input if there have any value except zero, example like S0:1 ; S1:0 ; S2:1 and insert into mysql just like this S0:1 ; S2:1 so if there no have value / zero, not being inserted. Thanks
You could try this:
String bs = "";
int count = 0;
for (int i = 0; i < size; i++) {
if(!s[i].equals(0.0)) {
if(count > 0) {
bs += " ; " ;
}
bs += "S" + count + ":" + s[i].split(".")[0];
count++;
}
}
try this code
private static String format (int x) {
if (x == 0) {
return null;
}
return String.format ("S%d = %d; ", x ,x);
}
// testing from main
{
StringBuilder buf = new StringBuilder();
for (int x = 0; x < 4; x++) { // dummy input stream
String rv = format (x);
if (rv != null) {
buf.append(rv);
}
}
System.out.println(buf.toString());
}
I'm trying to display 550 data points with periodic peaks (the flat line is 61). The problem is, that androidplot isn't drawing all the points correctly! From my log:
ECG I values 61,61,62,63,62,61,61,61,61,67,71,68,61,53,61,61,61,61,61,61,61,61,62,63,64,64,64,63,62,61,61,61
I've got the rangeboundaries set to plot.setRangeBoundaries(0,100, BoundaryMode.AUTO);, but as you can see, the peaks never drop to the 53 data point. I can see this lower point sometimes, but it gets smoothed out a fraction of a second later (as you can see in the screenshot).
My line and point formatter is:
LineAndPointFormatter lapf = new LineAndPointFormatter(p.color, null, null, null);
lapf.getLinePaint().setStrokeJoin(Paint.Join.MITER);
lapf.getLinePaint().setStrokeWidth(1);
I've tried with the both Paint.Join.ROUND and Paint.Join.BEVEL and got the same effect. I've also used the debugger to check that 53 is being inserted into the series.
EDIT
After some debugging, it looks like my pulse loop thread is wrong:
while (keepRunning) {
for (PulseXYSeries j : series) {
for (int k = 0; k < j.plotStep; k++) {
int at = (j.position + k) % j.getSize();
if (j.pulsing) {
if (j.pulsePosition == j.pulseValues.size() - 1) {
j.pulsing = false;
j.pulsePosition = 0;
} else {
try {
int pulseVal = j.pulseValues.get(j.pulsePosition);
j.setY(pulseVal,at);
j.pulsePosition += 1;
} catch(IndexOutOfBoundsException e) {
j.pulsePosition = 0;
}
}
} else {
j.setY(j.pulseValues.get(0), at);
long currTime = SystemClock.elapsedRealtime();
if (currTime - j.getLastPulse() >= j.getPulseDelay()) {
j.pulsing = true;
j.setLastPulse(currTime);
}
}
j.remove(((at + j.eraserSize) % j.getSize()));
}
j.position = (j.position + 1) % j.getSize(); // fixed it by changing +1 to + j.plotStep
}
Thread.sleep(delay);
}
My custom series looks like:
private class PulseXYSeries implements XYSeries {
private List<Integer> pulseValues = new ArrayList<Integer>();
private int pulsePerMinute;
public int pulsePosition;
public int position;
private ArrayList<Integer> values;
private String title;
private long lastPulse;
public boolean pulsing = false;
public int eraserSize = 20;
public int plotStep = 3;
}
I create a JTable with a customized model and characteristics. I have a long written class that carries out the set up and rendering properly. I see that the .getSelectedRows() method is not working and never evaluates to a value. I was trying to get the index of the first row selected and the last. Here is my code.
public void actionPerformed(ActionEvent event) {
String command = event.getActionCommand();
rows = table.getSelectedRows();
firstRow = rows[0];
int rowCount = rows.length;
lastRow = rows[(rowCount - 1)];
if (command.equals("Run Threw")) {
}else if (command.equals("Shield Bash")) {
this.attack(firstRow, lastRow, command);
}
public boolean block (int defendersRow) {
}
public boolean fumble (int attackersRow) {
}
public boolean dodge (int defendersRow) {
}
public boolean critical (int attackersRow, int attackRoll) {
}
public void attack(int firstRow, int lastRow, String command) {
command = this.command;
firstRow = this.firstRow;
lastRow = this.lastRow;
if (command == "Bludgeon" || command == "React" || command == "ShieldBash") {
attackersRow = this.lastRow;
defendersRow = this.firstRow;
}else if(command == "Attack" || command == "Skill") {
attackersRow = this.firstRow;
defendersRow = this.lastRow;
}else {
}
table.setValueAt(true, attackersRow, 16);
fumbled = this.fumble(attackersRow);
if (fumbled == true) {
outputString = "fumbled";
}
Object maxDamageObject = table.getValueAt(attackersRow, 10);
int maxDamage = (Integer) maxDamageObject;
attackRoll = generator.nextInt(100) + 1;
this.critical(attackersRow, attackRoll);
if (criticaled == true) {
outputString = "criticaled";
}
dodged = this.dodge(defendersRow);
if (dodged == true) {
outputString = "dodged";
}
blocked = this.block(defendersRow);
if (blocked == true) {
outputString = "blocked";
}
int defenseRoll = generator.nextInt(100) + 1;
Object attackBaseObject = table.getValueAt(attackersRow, 6);
Object defenseBaseObject = table.getValueAt(defendersRow, 11);
int attackBase = (Integer) attackBaseObject;
int defenseBase = (Integer) defenseBaseObject;
int attack = attackRoll + attackBase;
int defense = defenseRoll + defenseBase;
Object minDamageObject = table.getValueAt(attackersRow, 9);
int minDamage = (Integer) minDamageObject;
damage = generator.nextInt((maxDamage - minDamage))+minDamage;
if (criticaled == true) {
damage = maxDamage * 2;
}else if (attack >= (defense + 50)) {
damage = damage * 2;
}else if (attack >= defense) {
damage = damage;
}else {
damage = 0;
}
this.outputSelection(outputString, attackersRow, defendersRow, command, damage);
this.damage(defendersRow, damage);
}
private void damage(int defendersRow, int damage) {
}
private void outputSelection(String outputString, int attackersRow, int defendersRow, String command, int damage) {
}
I doesn't see problem with code. May be you can cross check following points:
Are you able to select single / multiple row(s)? Or check what jTableObject.getRowSelectionAllowed() returns
Does your code gets called only after selecting rows. Your code should be executed after some rows are selected so that it can return you expected values.
Listener which calls above mentioned code, is it added to table.
Small points but better to cross check :).
As shown in this working example, you can add a ListSelectionListener to your table's ListSelectionModel and see if anything comes through. It may help pin down where things have gone awry: for example, a local table reference that shadows the intended one.
I have a big text file (5Mb) that I use in my Android application. I create the file as a list of pre-sorted Strings, and the file doesn't change once it is created. How can I perform a binary search on the contents of this file, without reading line-by-line to find the matching String?
Since the content of the file does not change, you can break the file into multiple pieces. Say A-G, H-N, 0-T and U-Z. This allows you to check the first character and immediately be able to cut the possible set to a fourth of the original size. Now a linear search will not take as long or reading the whole file could be an option. This process could be extended if n/4 is still too large, but the idea is the same. Build the search breakdowns into the file structure instead of trying to do it all in memory.
A 5MB file isn't that big - you should be able to read each line into a String[] array, which you can then use java.util.Arrays.binarySearch() to find the line you want. This is my recommended approach.
If you don't want to read the whole file in to your app, then it gets more complicated. If each line of the file is the same length, and the file is already sorted, then you can open the file in RandomAccessFile and perform a binary search yourself by using seek() like this...
// open the file for reading
RandomAccessFile raf = new RandomAccessFile("myfile.txt","r");
String searchValue = "myline";
int lineSize = 50;
int numberOfLines = raf.length() / lineSize;
// perform the binary search...
byte[] lineBuffer = new byte[lineSize];
int bottom = 0;
int top = numberOfLines;
int middle;
while (bottom <= top){
middle = (bottom+top)/2;
raf.seek(middle*lineSize); // jump to this line in the file
raf.read(lineBuffer); // read the line from the file
String line = new String(lineBuffer); // convert the line to a String
int comparison = line.compareTo(searchValue);
if (comparison == 0){
// found it
break;
}
else if (comparison < 0){
// line comes before searchValue
bottom = middle + 1;
}
else {
// line comes after searchValue
top = middle - 1;
}
}
raf.close(); // close the file when you're finished
However, if the file doesn't have fixed-width lines, then you can't easily perform a binary search without loading it into memory first, as you can't quickly jump to a specific line in the file like you can with fixed-width lines.
Here's something I quickly put together. It uses two files, one with the words, the other with the offsets. The format of the offset file is this: the first 10 bits contains the word size, the last 22 bits contains the offset (the word position, for example, aaah would be 0, abasementable would be 4, etc.). It's encoded in big endian (java standard). Hope it helps somebody.
word.dat:
aaahabasementableabnormalabnormalityabortionistabortion-rightsabracadabra
wordx.dat:
00 80 00 00 01 20 00 04 00 80 00 0D 01 00 00 11 _____ __________
01 60 00 19 01 60 00 24 01 E0 00 2F 01 60 00 3E _`___`_$___/_`_>
I created these files in C#, but here's the code for it (it uses a txt file with words separated by crlfs)
static void Main(string[] args)
{
const string fIn = #"C:\projects\droid\WriteFiles\input\allwords.txt";
const string fwordxOut = #"C:\projects\droid\WriteFiles\output\wordx.dat";
const string fWordOut = #"C:\projects\droid\WriteFiles\output\word.dat";
int i = 0;
int offset = 0;
int j = 0;
var lines = File.ReadLines(fIn);
FileStream stream = new FileStream(fwordxOut, FileMode.Create, FileAccess.ReadWrite);
using (EndianBinaryWriter wwordxOut = new EndianBinaryWriter(EndianBitConverter.Big, stream))
{
using (StreamWriter wWordOut = new StreamWriter(File.Open(fWordOut, FileMode.Create)))
{
foreach (var line in lines)
{
wWordOut.Write(line);
i = offset | ((int)line.Length << 22); //first 10 bits to the left is the word size
offset = offset + (int)line.Length;
wwordxOut.Write(i);
//if (j == 7)
// break;
j++;
}
}
}
}
And this is the Java code for the binary file search:
public static void binarySearch() {
String TAG = "TEST";
String wordFilePath = Environment.getExternalStorageDirectory().getAbsolutePath() + "/word.dat";
String wordxFilePath = Environment.getExternalStorageDirectory().getAbsolutePath() + "/wordx.dat";
String target = "abracadabra";
boolean targetFound = false;
int searchCount = 0;
try {
RandomAccessFile raf = new RandomAccessFile(wordxFilePath, "r");
RandomAccessFile rafWord = new RandomAccessFile(wordFilePath, "r");
long low = 0;
long high = (raf.length() / 4) - 1;
int cur = 0;
long wordOffset = 0;
int len = 0;
while (high >= low) {
long mid = (low + high) / 2;
raf.seek(mid * 4);
cur = raf.readInt();
Log.v(TAG + "-cur", String.valueOf(cur));
len = cur >> 22; //word length
cur = cur & 0x3FFFFF; //first 10 bits are 0
rafWord.seek(cur);
byte [] bytes = new byte[len];
wordOffset = rafWord.read(bytes, 0, len);
Log.v(TAG + "-wordOffset", String.valueOf(wordOffset));
searchCount++;
String str = new String(bytes);
Log.v(TAG, str);
if (target.compareTo(str) < 0) {
high = mid - 1;
} else if (target.compareTo(str) == 0) {
targetFound = true;
break;
} else {
low = mid + 1;
}
}
raf.close();
rafWord.close();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
if (targetFound == true) {
Log.v(TAG + "-found " , String.valueOf(searchCount));
} else {
Log.v(TAG + "-not found " , String.valueOf(searchCount));
}
}
In a uniform character length text file you could seek to the middle of the interval in question character wise, start reading characters until you hit your deliminator, then use the subsequent string as an approximation for the element wise middle. The problem with doing this in android, though, is you apparently can't get random access to a resource (although I suppose you could just reopen it every time). Furthermore this technique doesn't generalize to maps and sets of other types.
Another option would be to (using a RandomAccessFile) write an "array" of ints - one for each String - at the beginning of the file then go back and update them with the locations of their corresponding Strings. Again the search will require jumping around.
What I would do (and did do in my own app) is implement a hash set in a file. This one does separate chaining with trees.
import java.io.BufferedInputStream;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedList;
import java.util.Set;
class StringFileSet {
private static final double loadFactor = 0.75;
public static void makeFile(String fileName, String comment, Set<String> set) throws IOException {
new File(fileName).delete();
RandomAccessFile fout = new RandomAccessFile(fileName, "rw");
//Write comment
fout.writeUTF(comment);
//Make bucket array
int numBuckets = (int)(set.size()/loadFactor);
ArrayList<ArrayList<String>> bucketArray = new ArrayList<ArrayList<String>>(numBuckets);
for (int ii = 0; ii < numBuckets; ii++){
bucketArray.add(new ArrayList<String>());
}
for (String key : set){
bucketArray.get(Math.abs(key.hashCode()%numBuckets)).add(key);
}
//Sort key lists in preparation for creating trees
for (ArrayList<String> keyList : bucketArray){
Collections.sort(keyList);
}
//Make queues in preparation for creating trees
class NodeInfo{
public final int lower;
public final int upper;
public final long callingOffset;
public NodeInfo(int lower, int upper, long callingOffset){
this.lower = lower;
this.upper = upper;
this.callingOffset = callingOffset;
}
}
ArrayList<LinkedList<NodeInfo>> queueList = new ArrayList<LinkedList<NodeInfo>>(numBuckets);
for (int ii = 0; ii < numBuckets; ii++){
queueList.add(new LinkedList<NodeInfo>());
}
//Write bucket array
fout.writeInt(numBuckets);
for (int index = 0; index < numBuckets; index++){
queueList.get(index).add(new NodeInfo(0, bucketArray.get(index).size()-1, fout.getFilePointer()));
fout.writeInt(-1);
}
//Write trees
for (int bucketIndex = 0; bucketIndex < numBuckets; bucketIndex++){
while (queueList.get(bucketIndex).size() != 0){
NodeInfo nodeInfo = queueList.get(bucketIndex).poll();
if (nodeInfo.lower <= nodeInfo.upper){
//Set respective pointer in parent node
fout.seek(nodeInfo.callingOffset);
fout.writeInt((int)(fout.length() - (nodeInfo.callingOffset + 4))); //Distance instead of absolute position so that the get method can use a DataInputStream
fout.seek(fout.length());
int middle = (nodeInfo.lower + nodeInfo.upper)/2;
//Key
fout.writeUTF(bucketArray.get(bucketIndex).get(middle));
//Left child
queueList.get(bucketIndex).add(new NodeInfo(nodeInfo.lower, middle-1, fout.getFilePointer()));
fout.writeInt(-1);
//Right child
queueList.get(bucketIndex).add(new NodeInfo(middle+1, nodeInfo.upper, fout.getFilePointer()));
fout.writeInt(-1);
}
}
}
fout.close();
}
private final String fileName;
private final int numBuckets;
private final int bucketArrayOffset;
public StringFileSet(String fileName) throws IOException {
this.fileName = fileName;
DataInputStream fin = new DataInputStream(new BufferedInputStream(new FileInputStream(fileName)));
short numBytes = fin.readShort();
fin.skipBytes(numBytes);
this.numBuckets = fin.readInt();
this.bucketArrayOffset = numBytes + 6;
fin.close();
}
public boolean contains(String key) throws IOException {
boolean containsKey = false;
DataInputStream fin = new DataInputStream(new BufferedInputStream(new FileInputStream(this.fileName)));
fin.skipBytes(4*(Math.abs(key.hashCode()%this.numBuckets)) + this.bucketArrayOffset);
int distance = fin.readInt();
while (distance != -1){
fin.skipBytes(distance);
String candidate = fin.readUTF();
if (key.compareTo(candidate) < 0){
distance = fin.readInt();
}else if (key.compareTo(candidate) > 0){
fin.skipBytes(4);
distance = fin.readInt();
}else{
fin.skipBytes(8);
containsKey = true;
break;
}
}
fin.close();
return containsKey;
}
}
A test program
import java.io.File;
import java.io.IOException;
import java.util.HashSet;
class Test {
public static void main(String[] args) throws IOException {
HashSet<String> stringMemorySet = new HashSet<String>();
stringMemorySet.add("red");
stringMemorySet.add("yellow");
stringMemorySet.add("blue");
StringFileSet.makeFile("stringSet", "Provided under ... included in all copies and derivatives ...", stringMemorySet);
StringFileSet stringFileSet = new StringFileSet("stringSet");
System.out.println("orange -> " + stringFileSet.contains("orange"));
System.out.println("red -> " + stringFileSet.contains("red"));
System.out.println("yellow -> " + stringFileSet.contains("yellow"));
System.out.println("blue -> " + stringFileSet.contains("blue"));
new File("stringSet").delete();
System.out.println();
}
}
You'll also need to pass a Context to it, if and when you modify it for android, so it can access the getResources() method.
You're also probably going to want to stop the android build tools from compressing the file, which can apparently only be done - if you're working with the GUI - by changing the file's extension to something such as jpg. This made the process about 100 to 300 times faster in my app.
You might also look into giving yourself more memory by using the NDK.
Though it might sound like overkill, don't store data you need to do this with as a flat file. Make a database and query the data in the database. This should be both effective and fast.
Here is a function that I think works (using this in practice). Lines can have any length. You have to supply a lambda called "nav" to do the actual line check so you are flexible in the file's order (case-sensitive, case-insensitive, ordered by a certain field etc.).
import java.io.File;
import java.io.RandomAccessFile;
class main {
// returns pair(character range in file, line) or null if not found
// if no exact match found, return line above
// nav takes a line and returns -1 (move up), 0 (found) or 1 (move down)
// The line supplied to nav is stripped of the trailing \n, but not the \r
// UTF-8 encoding is assumed
static Pair<LongRange, String> binarySearchForLineInTextFile(File file, IF1<String, Integer> nav) {
long length = l(file);
int bufSize = 1024;
RandomAccessFile raf = randomAccessFileForReading(file);
try {
long min = 0, max = length;
int direction = 0;
Pair<LongRange, String> possibleResult = null;
while (min < max) {
ping();
long middle = (min + max) / 2;
long lineStart = raf_findBeginningOfLine(raf, middle, bufSize);
long lineEnd = raf_findEndOfLine(raf, middle, bufSize);
String line = fromUtf8(raf_readFilePart(raf, lineStart, (int) (lineEnd - 1 - lineStart)));
direction = nav.get(line);
possibleResult = (Pair<LongRange, String>) new Pair(new LongRange(lineStart, lineEnd), line);
if (direction == 0) return possibleResult;
// asserts are to assure that loop terminates
if (direction < 0) max = assertLessThan(max, lineStart);
else min = assertBiggerThan(min, lineEnd);
}
if (direction >= 0) return possibleResult;
long lineStart = raf_findBeginningOfLine(raf, min - 1, bufSize);
String line = fromUtf8(raf_readFilePart(raf, lineStart, (int) (min - 1 - lineStart)));
return new Pair(new LongRange(lineStart, min), line);
} finally {
_close(raf);
}
}
static int l(byte[] a) {
return a == null ? 0 : a.length;
}
static long l(File f) {
return f == null ? 0 : f.length();
}
static RandomAccessFile randomAccessFileForReading(File path) {
try {
return new RandomAccessFile(path, "r");
} catch (Exception __e) {
throw rethrow(__e);
}
}
// you can change this function to allow interrupting long calculations from the outside. just throw a RuntimeException.
static boolean ping() {
return true;
}
static long raf_findBeginningOfLine(RandomAccessFile raf, long pos, int bufSize) {
try {
byte[] buf = new byte[bufSize];
while (pos > 0) {
long start = Math.max(pos - bufSize, 0);
raf.seek(start);
raf.readFully(buf, 0, (int) Math.min(pos - start, bufSize));
int idx = lastIndexOf_byteArray(buf, (byte) '\n');
if (idx >= 0) return start + idx + 1;
pos = start;
}
return 0;
} catch (Exception __e) {
throw rethrow(__e);
}
}
static long raf_findEndOfLine(RandomAccessFile raf, long pos, int bufSize) {
try {
byte[] buf = new byte[bufSize];
long length = raf.length();
while (pos < length) {
raf.seek(pos);
raf.readFully(buf, 0, (int) Math.min(length - pos, bufSize));
int idx = indexOf_byteArray(buf, (byte) '\n');
if (idx >= 0) return pos + idx + 1;
pos += bufSize;
}
return length;
} catch (Exception __e) {
throw rethrow(__e);
}
}
static String fromUtf8(byte[] bytes) {
try {
return bytes == null ? null : new String(bytes, "UTF-8");
} catch (Exception __e) {
throw rethrow(__e);
}
}
static byte[] raf_readFilePart(RandomAccessFile raf, long start, int l) {
try {
byte[] buf = new byte[l];
raf.seek(start);
raf.readFully(buf);
return buf;
} catch (Exception __e) {
throw rethrow(__e);
}
}
static <A> A assertLessThan(A a, A b) {
assertTrue(cmp(b, a) < 0);
return b;
}
static <A> A assertBiggerThan(A a, A b) {
assertTrue(cmp(b, a) > 0);
return b;
}
static void _close(AutoCloseable c) {
try {
if (c != null)
c.close();
} catch (Throwable e) {
throw rethrow(e);
}
}
static RuntimeException rethrow(Throwable t) {
throw t instanceof RuntimeException ? (RuntimeException) t : new RuntimeException(t);
}
static int lastIndexOf_byteArray(byte[] a, byte b) {
for (int i = l(a) - 1; i >= 0; i--)
if (a[i] == b)
return i;
return -1;
}
static int indexOf_byteArray(byte[] a, byte b) {
int n = l(a);
for (int i = 0; i < n; i++)
if (a[i] == b)
return i;
return -1;
}
static boolean assertTrue(boolean b) {
if (!b)
throw fail("oops");
return b;
}
static int cmp(Object a, Object b) {
if (a == null) return b == null ? 0 : -1;
if (b == null) return 1;
return ((Comparable) a).compareTo(b);
}
static RuntimeException fail(String msg) {
throw new RuntimeException(msg == null ? "" : msg);
}
final static class LongRange {
long start, end;
LongRange(long start, long end) {
this.end = end;
this.start = start;
}
public String toString() {
return "[" + start + ";" + end + "]";
}
}
interface IF1<A, B> {
B get(A a);
}
static class Pair<A, B> {
A a;
B b;
Pair(A a, B b) {
this.b = b;
this.a = a;
}
public String toString() {
return "<" + a + ", " + b + ">";
}
}
}