Inserting records into Berkeley DB with Java

Inserting records into Berkeley DB with Java - java

I can't seem to find sample code for constructing a Berkeley DB in Java and inserting records into it. Any samples? And I do not mean the Berkeley DB Java Edition either.

http://www.oracle.com/technology/documentation/berkeley-db/db/programmer_reference/BDB_Prog_Reference.pdf
Chapter 5
If you download db-5.0.21.NC.zip you will see plenty of samples.
Here is one that seems to do what you want
/*-
* See the file LICENSE for redistribution information.
*
* Copyright (c) 2004, 2010 Oracle and/or its affiliates. All rights reserved.
*
* $Id$
*/
// File: ExampleDatabaseLoad.java
package db.GettingStarted;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
import java.util.StringTokenizer;
import java.util.Vector;
import com.sleepycat.bind.EntryBinding;
import com.sleepycat.bind.serial.SerialBinding;
import com.sleepycat.bind.tuple.TupleBinding;
import com.sleepycat.db.DatabaseEntry;
import com.sleepycat.db.DatabaseException;
public class ExampleDatabaseLoad {
private static String myDbsPath = "./";
private static File inventoryFile = new File("./inventory.txt");
private static File vendorsFile = new File("./vendors.txt");
// DatabaseEntries used for loading records
private static DatabaseEntry theKey = new DatabaseEntry();
private static DatabaseEntry theData = new DatabaseEntry();
// Encapsulates the databases.
private static MyDbs myDbs = new MyDbs();
private static void usage() {
System.out.println("ExampleDatabaseLoad [-h <database home>]");
System.out.println(" [-i <inventory file>] [-v <vendors file>]");
System.exit(-1);
}
public static void main(String args[]) {
ExampleDatabaseLoad edl = new ExampleDatabaseLoad();
try {
edl.run(args);
} catch (DatabaseException dbe) {
System.err.println("ExampleDatabaseLoad: " + dbe.toString());
dbe.printStackTrace();
} catch (Exception e) {
System.out.println("Exception: " + e.toString());
e.printStackTrace();
} finally {
myDbs.close();
}
System.out.println("All done.");
}
private void run(String args[])
throws DatabaseException {
// Parse the arguments list
parseArgs(args);
myDbs.setup(myDbsPath);
System.out.println("loading vendors db....");
loadVendorsDb();
System.out.println("loading inventory db....");
loadInventoryDb();
}
private void loadVendorsDb()
throws DatabaseException {
// loadFile opens a flat-text file that contains our data
// and loads it into a list for us to work with. The integer
// parameter represents the number of fields expected in the
// file.
List vendors = loadFile(vendorsFile, 8);
// Now load the data into the database. The vendor's name is the
// key, and the data is a Vendor class object.
// Need a serial binding for the data
EntryBinding dataBinding =
new SerialBinding(myDbs.getClassCatalog(), Vendor.class);
for (int i = 0; i < vendors.size(); i++) {
String[] sArray = (String[])vendors.get(i);
Vendor theVendor = new Vendor();
theVendor.setVendorName(sArray[0]);
theVendor.setAddress(sArray[1]);
theVendor.setCity(sArray[2]);
theVendor.setState(sArray[3]);
theVendor.setZipcode(sArray[4]);
theVendor.setBusinessPhoneNumber(sArray[5]);
theVendor.setRepName(sArray[6]);
theVendor.setRepPhoneNumber(sArray[7]);
// The key is the vendor's name.
// ASSUMES THE VENDOR'S NAME IS UNIQUE!
String vendorName = theVendor.getVendorName();
try {
theKey = new DatabaseEntry(vendorName.getBytes("UTF-8"));
} catch (IOException willNeverOccur) {}
// Convert the Vendor object to a DatabaseEntry object
// using our SerialBinding
dataBinding.objectToEntry(theVendor, theData);
// Put it in the database.
myDbs.getVendorDB().put(null, theKey, theData);
}
}
private void loadInventoryDb()
throws DatabaseException {
// loadFile opens a flat-text file that contains our data
// and loads it into a list for us to work with. The integer
// parameter represents the number of fields expected in the
// file.
List inventoryArray = loadFile(inventoryFile, 6);
// Now load the data into the database. The item's sku is the
// key, and the data is an Inventory class object.
// Need a tuple binding for the Inventory class.
TupleBinding inventoryBinding = new InventoryBinding();
for (int i = 0; i < inventoryArray.size(); i++) {
String[] sArray = (String[])inventoryArray.get(i);
String sku = sArray[1];
try {
theKey = new DatabaseEntry(sku.getBytes("UTF-8"));
} catch (IOException willNeverOccur) {}
Inventory theInventory = new Inventory();
theInventory.setItemName(sArray[0]);
theInventory.setSku(sArray[1]);
theInventory.setVendorPrice((new Float(sArray[2])).floatValue());
theInventory.setVendorInventory((new Integer(sArray[3])).intValue());
theInventory.setCategory(sArray[4]);
theInventory.setVendor(sArray[5]);
// Place the Vendor object on the DatabaseEntry object using our
// the tuple binding we implemented in InventoryBinding.java
inventoryBinding.objectToEntry(theInventory, theData);
// Put it in the database. Note that this causes our secondary database
// to be automatically updated for us.
myDbs.getInventoryDB().put(null, theKey, theData);
}
}
private static void parseArgs(String args[]) {
for(int i = 0; i < args.length; ++i) {
if (args[i].startsWith("-")) {
switch(args[i].charAt(1)) {
case 'h':
myDbsPath = new String(args[++i]);
break;
case 'i':
inventoryFile = new File(args[++i]);
break;
case 'v':
vendorsFile = new File(args[++i]);
break;
default:
usage();
}
}
}
}
private List loadFile(File theFile, int numFields) {
List records = new ArrayList();
try {
String theLine = null;
FileInputStream fis = new FileInputStream(theFile);
BufferedReader br = new BufferedReader(new InputStreamReader(fis));
while((theLine=br.readLine()) != null) {
String[] theLineArray = splitString(theLine, "#");
if (theLineArray.length != numFields) {
System.out.println("Malformed line found in " + theFile.getPath());
System.out.println("Line was: '" + theLine);
System.out.println("length found was: " + theLineArray.length);
System.exit(-1);
}
records.add(theLineArray);
}
fis.close();
} catch (FileNotFoundException e) {
System.err.println(theFile.getPath() + " does not exist.");
e.printStackTrace();
usage();
} catch (IOException e) {
System.err.println("IO Exception: " + e.toString());
e.printStackTrace();
System.exit(-1);
}
return records;
}
private static String[] splitString(String s, String delimiter) {
Vector resultVector = new Vector();
StringTokenizer tokenizer = new StringTokenizer(s, delimiter);
while (tokenizer.hasMoreTokens())
resultVector.add(tokenizer.nextToken());
String[] resultArray = new String[resultVector.size()];
resultVector.copyInto(resultArray);
return resultArray;
}
protected ExampleDatabaseLoad() {}
}

There are a number of good Getting Started Guides for Oracle Berkeley DB Java Edition. They are included in the documentation set. You'll find example code in the documentation. If you're new to Oracle Berkeley DB Java Edition that's the right place to start. There are other examples in the download package as well.
I'm the product manager for Oracle Berkeley DB, I hope this addressed your question. If not please let me know how else I can help you.

Related

How to do HBase range scan for Hexadecimal row key?

The following worked in HBase shell, when try to perform range scan on HBase shell.
scan 'mytable', {STARTROW => "\x00\x00\x00\x00\x01\x8F\xF6\x83", ENDROW => "\x00\x00\x00\x00\x01\x8F\xF6\x8D"}
But when try to implement Java client to perform the same, it retrieves no result.
Scan scan = new Scan(Bytes.ToBytes("\x00\x00\x00\x00\x01\x8F\xF6\x83"),Bytes.toBytes("\x00\x00\x00\x00\x01\x8F\xF6\x8D");
scan.setFilter(colFilter);
scan.setOtherStuff...
ResultScanner scanner = table.getScanner(scan);
for (Result result = scanner.next(); result != null; result = scanner.next()) {
....
}
I tried to escape "\" character and pass the start and end row keys. But it didn't work as expected.
I'm passing the input data as command line arguments.
time java -jar $ARIADNE3D_CLI PCRangeSearchTxt -table_name $TABLE_NAME -m 4 -start_key "\x00\x00\x00\x00\x01\x8F\xF6\x8D" -end_key "\x00\x00\x00\x00\x01\x8F\xF6\x8D" -o $SCRATCH/txt-1.txt
The Java implementation for PCRangeSearchTxt is as follows
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package umg.ariadne3d.core.query.pc;
import java.io.*;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.PosixParser;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import umg.ariadne3d.core.common.Constants;
import umg.core.common.Executable;
/**
* Point cloud range search.
* #author VVo
*/
public class PCRangeSearchTxt implements Executable {
static Logger LOGGER = Logger.getLogger(PCRangeSearchTxt.class);
public static final String NAME = "PCRANGESEARCHTXT"; //PCRangeSearchTxt
public static void main(String[] args) {
args = new String[]{
// "-t", "d15-tiny-m4",
// "-m", "4",
// "-index", "/Users/vu/scratch/ariadne3d/pointcloud/meta/hilbert.json",
// "-query", "/Users/vu/scratch/ariadne3d/query/q0.json",
// "-las_meta", "/Users/vu/scratch/ariadne3d/pointcloud/meta/d15-meta.json",
// "-o", "/Users/vu/tmp/a.las"
};
Executable prog = new PCRangeSearchTxt();
int err = prog.run(args);
System.exit(err);
}
#Override
public int run(String[] args) {
CommandLine cmd = parseArgs(args);
String tableName = cmd.getOptionValue("t");
String start_key = cmd.getOptionValue("start_key");
String end_key = cmd.getOptionValue("end_key");
final String FILENAME = cmd.getOptionValue("o");
int modelNo = Integer.parseInt(cmd.getOptionValue("m"));
try{
File file = new File(FILENAME);
// if file doesnt exists, then create it
if (!file.exists()) {
file.createNewFile();
}
}catch (IOException e) {
e.printStackTrace();
}
Configuration conf = HBaseConfiguration.create();
String[] connectionParams = null;
if (cmd.hasOption("conn")) {
connectionParams = cmd.getOptionValues("conn");
}
if (connectionParams != null) {
conf.set(Constants.HBASE_CONFIGURATION_ZOOKEEPER_QUORUM, connectionParams[0]);
LOGGER.debug(String.format("Set quorum string %s", conf.get(Constants.HBASE_CONFIGURATION_ZOOKEEPER_QUORUM)));
conf.setInt(Constants.HBASE_CONFIGURATION_ZOOKEEPER_CLIENTPORT, Integer.parseInt(connectionParams[1]));
LOGGER.debug(String.format("Set port %d", conf.getInt(Constants.HBASE_CONFIGURATION_ZOOKEEPER_CLIENTPORT, 0)));
}
try {
long start = System.currentTimeMillis();
Connection connection = ConnectionFactory.createConnection(conf);
HBaseConfiguration.addHbaseResources(conf);
Table table = connection.getTable(TableName.valueOf(tableName));
byte[] keyStart = Bytes.toBytes(start_key);
byte[] keyEnd = Bytes.toBytes(end_key);
Scan scan = new Scan(keyStart, keyEnd);
ResultScanner scanner = table.getScanner(scan);
FileWriter writer = new FileWriter(FILENAME, true);
try{
for (Result result = scanner.next(); result != null; result = scanner.next()) {
writer.write(result.toString()+"\n");
}
}finally {
writer.close();
scanner.close();
}
long end = System.currentTimeMillis();
System.out.printf("Total time %d \n", end - start);
table.close();
connection.close();
return 0;
} catch (IOException ex) {
LOGGER.error(ex);
return 1;
}
}
private static CommandLine parseArgs(String[] args) {
Options options = new Options();
Option o;
// table name
o = new Option("t","table_name", true, "HBase table name");
options.addOption(o);
o = new Option("m", "model_number", true, "model number");
options.addOption(o);
o = new Option("start_key", true, "start key for range scan");
options.addOption(o);
o = new Option("end_key", true, "end key for range scan");
options.addOption(o);
o = new Option("o", "output", true, "create output file");
o.setRequired(false);
options.addOption(o);
// connection parameters
o = new Option("conn", "connection", true, "Zookepper quorum and port");
o.setArgs(2);
o.setRequired(false);
options.addOption(o);
// debug flag
options.addOption("d", "debug", false, "switch on DEBUG log level");
CommandLineParser parser = new PosixParser();
CommandLine cmd = null;
try {
cmd = parser.parse(options, args);
} catch (Exception e) {
System.err.println("ERROR: " + e.getMessage() + "\n");
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp(NAME + " ", options, true);
System.exit(-1);
}
if (cmd.hasOption("d")) {
LOGGER.setLevel(Level.DEBUG);
System.out.println("DEBUG ON");
}
return cmd;
}
}
What is the right way of implementing HBase range search on row keys that are in hexadecimal ?

I would imagine that you know what is used as a key in your HBase table so I don't see why you can't do this:
byte[]start = Hex.decodeHex("startKey".toCharArray());
byte[]end = Hex.decodeHex("endKey".toCharArray());
Scan scan = new Scan(start, end)
Just not sure why you are trying to do this the other way around.
Otherwise there is an answer to your question here:
What are the non-hex characters in HBase Shell RowKey?
Hope that helps :)

I was able to find a solution for this problem. Basically what I have done in the code is that, I've declared the row keys as String variables. As I'm passing start_key and end_key as command line arguments and HBase internally stores data in byte array[] sequence, I could have simply pass the start key in a known format, that is I need not to pass the value as a hex code. For example I could pass both start_key and end_key in their original form/human readable form and HBase will map this form into the HBase internal byte array[] form.
Therefore, I modified the above Java class to accept start_key and end_key as double data type values.
// key format: 392994.475499
double startTS = Double.parseDouble(cmd.getOptionValue("start_key"));
double endTS = Double.parseDouble(cmd.getOptionValue("end_key"));
On summery, it was a matter of understanding the internal structure of HBase and accepting the command line values as double values.
By doing so I was able to run the code as expected.
For you all to get a complete idea of what I did, below I shared the modified code of the class.
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package umg.ariadne3d.core.query.pc;
import java.io.*;
import java.nio.file.Paths;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import umg.ariadne3d.core.store.schema.pc.Model4;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.PosixParser;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import umg.ariadne3d.core.common.Constants;
import umg.core.common.Executable;
/**
* Point cloud range search.
* #author
*/
public class PCMultiClientRangeSearchSQN implements Executable {
static Logger LOGGER = Logger.getLogger(PCMultiClientRangeSearchSQN.class);
public static final String NAME = "PCMULTICLIENTRANGESEARCHSQN"; //PCMultiClientRangeSearchSQN
public static void main(String[] args) {
args = new String[]{
// "-t", "d15-tiny-m4",
// "-m", "4",
// "-index", "/Users/vu/scratch/ariadne3d/pointcloud/meta/hilbert.json",
// "-query", "/Users/vu/scratch/ariadne3d/query/q0.json",
// "-las_meta", "/Users/vu/scratch/ariadne3d/pointcloud/meta/d15-meta.json",
// "-o", "/Users/vu/tmp/a.las"
};
Executable prog = new PCMultiClientRangeSearchSQN();
int err = prog.run(args);
System.exit(err);
}
#Override
public int run(String[] args) {
CommandLine cmd = parseArgs(args);
String tableName = cmd.getOptionValue("t");
// key format: 392994.475499
double startTS = Double.parseDouble(cmd.getOptionValue("start_key"));
double endTS = Double.parseDouble(cmd.getOptionValue("end_key"));
long startRowkey = Math.round((startTS - 388800) / 0.000001);
long endRowkey = Math.round((endTS - 388800) / 0.000001);
int numOfClients;
if (cmd.hasOption("clients")) {
numOfClients = Integer.parseInt(cmd.getOptionValue("clients"));
} else {
numOfClients = 1;//Runtime.getRuntime().availableProcessors();
}
//System.out.println(numOfClients);
final String FILENAME = cmd.getOptionValue("o");
final String FILEPATH = new File("").getAbsolutePath();
for(int i=0; i<numOfClients; i++){
try{
File file = new File(FILENAME+i+".txt");
// if file doesnt exists, then create it
if (!file.exists()) {
file.createNewFile();
}
}catch (IOException e) {
e.printStackTrace();
}
}
Configuration conf = HBaseConfiguration.create();
String[] connectionParams = null;
if (cmd.hasOption("conn")) {
connectionParams = cmd.getOptionValues("conn");
}
if (connectionParams != null) {
conf.set(Constants.HBASE_CONFIGURATION_ZOOKEEPER_QUORUM, connectionParams[0]);
LOGGER.debug(String.format("Set quorum string %s", conf.get(Constants.HBASE_CONFIGURATION_ZOOKEEPER_QUORUM)));
conf.setInt(Constants.HBASE_CONFIGURATION_ZOOKEEPER_CLIENTPORT, Integer.parseInt(connectionParams[1]));
LOGGER.debug(String.format("Set port %d", conf.getInt(Constants.HBASE_CONFIGURATION_ZOOKEEPER_CLIENTPORT, 0)));
}
try {
Connection connection = ConnectionFactory.createConnection(conf);
HBaseConfiguration.addHbaseResources(conf);
Table table = connection.getTable(TableName.valueOf(tableName));
byte[] keyStart = Bytes.toBytes(startRowkey);
byte[] keyEnd = Bytes.toBytes(endRowkey);
ExecutorService executorService = Executors.newFixedThreadPool(100);
//submit the range scan task for execution
for(int j =0; j< numOfClients; j++){
executorService.execute(new RangeScan(keyStart, keyEnd, table, new File(FILEPATH+"/"+FILENAME+j+".txt")));
}
executorService.shutdown();
System.out.println("-----------------------");
// wait until all tasks are finished
try{
executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
}catch (Exception e){
System.out.println("Error ");
}
System.out.println("All tasks are finished!");
table.close();
connection.close();
return 0;
} catch (IOException ex) {
LOGGER.error(ex);
return 1;
}
}
class RangeScan implements Runnable{
byte[] keyStart;
byte[] keyEnd;
Table table;
File file;
RangeScan(byte[] keyStart, byte[] keyEnd, Table table, File file){
this.keyStart = keyStart;
this.keyEnd = keyEnd;
this.table = table;
this.file = file;
}
#Override
public void run() {
long start = System.currentTimeMillis();
Scan scan = new Scan(keyStart, keyEnd);
try{
ResultScanner scanner = table.getScanner(scan);
FileWriter writer = new FileWriter(file, true);
try{
for (Result result = scanner.next(); result != null; result = scanner.next()) {
// System.out.println("result "+result.toString());
//byte[] rawPointBytes = result.getValue(Model4.RAW_SENSING_DATA_FAM, Model4.POINT_COL);
//LASPointProtos.LASPointP pointP = LASPointProtos.LASPointP.parseFrom(rawPointBytes);
writer.write(result.toString()+"\n");
}
}finally {
writer.close();
scanner.close();
}
long end = System.currentTimeMillis();
System.out.printf("Total time For File %s is %d \n", file.toString(),end - start);
}
catch (Exception ex){
LOGGER.error(ex);
}
// return 0;
}
}
private static CommandLine parseArgs(String[] args) {
Options options = new Options();
Option o;
// table name
o = new Option("t","table_name", true, "HBase table name");
options.addOption(o);
o = new Option("start_key", true, "start key for range scan");
options.addOption(o);
o = new Option("end_key", true, "end key for range scan");
options.addOption(o);
o = new Option("clients", true, "number of concurrent clients");
options.addOption(o);
o = new Option("o", "output", true, "create output file");
o.setRequired(false);
options.addOption(o);
// connection parameters
o = new Option("conn", "connection", true, "Zookepper quorum and port");
o.setArgs(2);
o.setRequired(false);
options.addOption(o);
// debug flag
options.addOption("d", "debug", false, "switch on DEBUG log level");
CommandLineParser parser = new PosixParser();
CommandLine cmd = null;
try {
cmd = parser.parse(options, args);
} catch (Exception e) {
System.err.println("ERROR: " + e.getMessage() + "\n");
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp(NAME + " ", options, true);
System.exit(-1);
}
if (cmd.hasOption("d")) {
LOGGER.setLevel(Level.DEBUG);
System.out.println("DEBUG ON");
}
return cmd;
}
}

Unable to access method in arraylist class from other classes in java

I have 2 class files in my simple project - sorry another newbee here!
But I get a compilation error on the last part where I am trying to print the hopefully stored configuration settings from a file for my project that will be referred to throughout the project.
The file is just rows of values like this 'ButtonConfig,8,V,NULL,bunny,mpg'
I basically want to be able to used the contents of this arraylist to dynamicly set up the configuration of a Raspberry pi GPO pins i.e. for the above values button attached to GPO pin 8 will play video (V) "<..other value...>_bunny.mpg"
Any help greatly appreciated - just telling me why I can't access the getExtension method would be nice!
Contents of first java file is -
package bpunit;
public class ButtonConfig {
private String keyword;
private String gponumber;
private String buttontype;
private String language;
private String filename;
private String extension;
public String getKeyword() {
return keyword;
}
public void setKeyword(String keyword) {
this.keyword = keyword;
}
...............
public String getExtension() {
return extension;
}
public void setExtension(String extension) {
this.extension = extension;
}
}
The second contains this -
package bpunit;
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
public class Read_ini {
public void Read_ini_toObject()
{
String csvFileToRead = "configs/BPUnit.properties";
BufferedReader br = null;
String line;
String splitBy = ",";
List buttonList = new ArrayList();
try {
br = new BufferedReader(new FileReader(csvFileToRead));
while ((line = br.readLine()) != null) {
// split on comma(',')
String[] buttonconfig = line.split(splitBy);
// create button object to store values
ButtonConfig buttonObject = new ButtonConfig();
// add values from csv to car object
buttonObject.setKeyword(buttonconfig[0]);
buttonObject.setGponumber(buttonconfig[1]);
buttonObject.setButtontype(buttonconfig[2]);
buttonObject.setLanguage(buttonconfig[3]);
buttonObject.setFilename(buttonconfig[4]);
buttonObject.setExtension(buttonconfig[5]);
// adding button object to a list
buttonList.add(buttonObject);
}
// print values stored in buttonList
printButtonList(buttonList);
} catch (FileNotFoundException e) {
System.out.print(e);
} catch (IOException e) {
System.out.print(e);
} finally {
if (br != null) {
try {
br.close();
} catch (IOException e) {
System.out.print(e);
}
}
}
}
public void printButtonList(List buttonListToPrint) {
for (int i = 0; i < buttonListToPrint.size(); i++) {
// THE LINE BELOW FAILS - getExtension() does not exist
// and all other attempts give me pointer references
//instead of the text //
System.out.println(buttonListToPrint.get(i).getExtension());
}
}
}

You have to add the parameterized type ButtonConfig to your ArrayList. It ends up being List<ButtonConfig> instead of just List.
package bpunit;
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
public class Read_ini {
public void Read_ini_toObject()
{
String csvFileToRead = "configs/BPUnit.properties";
BufferedReader br = null;
String line;
String splitBy = ",";
List<ButtonConfig> buttonList = new ArrayList<ButtonConfig>();
try {
br = new BufferedReader(new FileReader(csvFileToRead));
while ((line = br.readLine()) != null) {
// split on comma(',')
String[] buttonconfig = line.split(splitBy);
// create button object to store values
ButtonConfig buttonObject = new ButtonConfig();
// add values from csv to car object
buttonObject.setKeyword(buttonconfig[0]);
buttonObject.setGponumber(buttonconfig[1]);
buttonObject.setButtontype(buttonconfig[2]);
buttonObject.setLanguage(buttonconfig[3]);
buttonObject.setFilename(buttonconfig[4]);
buttonObject.setExtension(buttonconfig[5]);
// adding button object to a list
buttonList.add(buttonObject);
}
// print values stored in buttonList
printButtonList(buttonList);
} catch (FileNotFoundException e) {
System.out.print(e);
} catch (IOException e) {
System.out.print(e);
} finally {
if (br != null) {
try {
br.close();
} catch (IOException e) {
System.out.print(e);
}
}
}
}
public void printButtonList(List<ButtonConfig> buttonListToPrint) {
for (int i = 0; i < buttonListToPrint.size(); i++) {
// THE LINE BELOW FAILS - getExtension() does not exist
// and all other attempts give me pointer references
//instead of the text //
System.out.println(buttonListToPrint.get(i).getExtension());
}
}
}

The reason why the compilation is failing is because when you add an object to the ArrayList it is upcast as an object of the class Object. Now when you extract it you simply have to typecast it back to the original type. so all you have to do is this :
public void printButtonList(List buttonListToPrint) {
for (int i = 0; i < buttonListToPrint.size(); i++) {
// THE LINE BELOW FAILS - getExtension() does not exist
// and all other attempts give me pointer references
//instead of the text
ButtonConfig buttonObject =(ButtonConfig)buttonListToPrint.get(i);
System.out.println(buttonObject.getExtension());
}
}
Or as mentioned in the comments and answers above you could use generics and create an List of type ButtonConfig
public void Read_ini_toObject()
{
String csvFileToRead = "configs/BPUnit.properties";
BufferedReader br = null;
String line;
String splitBy = ",";
List<ButtonConfig> buttonList = new ArrayList<ButtonConfig>();
and pass it in the function printButtonList
public void printButtonList(List<ButtonConfig> buttonListToPrint) {
for (int i = 0; i < buttonListToPrint.size(); i++) {
// THE LINE BELOW FAILS - getExtension() does not exist
// and all other attempts give me pointer references
//instead of the text
System.out.println(buttonListToPrint.get(i).getExtension());
}
}

Hive, how do I retrieve all the database's tables columns

I want to write the equivalent of this sql request in Hive :
select * from information_schema.columns where table_schema='database_name'
How can I access hive's metastore and retrieve all the columns of all the tables stored in a specific database? I know that we can do
it by table via describe [table_name] but is there anyway to have all
the columns for all the tables in a database in the same request?

How can I access hive's metastore and retrieve all the columns of all
the tables stored in a specific database?
This is one way to connect HiveMetaStoreClient and you can use method getTableColumnsInformation will get columns.
In this class along with columns all the other information like partitions can be extracted. pls see example client and sample methods.
import org.apache.hadoop.hive.conf.HiveConf;
// test program
public class Test {
public static void main(String[] args){
HiveConf hiveConf = new HiveConf();
hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://host:port");
HiveMetaStoreConnector hiveMetaStoreConnector = new HiveMetaStoreConnector(hiveConf);
if(hiveMetaStoreConnector != null){
System.out.print(hiveMetaStoreConnector.getAllPartitionInfo("tablename"));
}
}
}
// define a class like this
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.thrift.TException;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
public class HiveMetaStoreConnector {
private HiveConf hiveConf;
HiveMetaStoreClient hiveMetaStoreClient;
public HiveMetaStoreConnector(String msAddr, String msPort){
try {
hiveConf = new HiveConf();
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, msAddr+":"+ msPort);
hiveMetaStoreClient = new HiveMetaStoreClient(hiveConf);
} catch (MetaException e) {
e.printStackTrace();
System.err.println("Constructor error");
System.err.println(e.toString());
System.exit(-100);
}
}
public HiveMetaStoreConnector(HiveConf hiveConf){
try {
this.hiveConf = hiveConf;
hiveMetaStoreClient = new HiveMetaStoreClient(hiveConf);
} catch (MetaException e) {
e.printStackTrace();
System.err.println("Constructor error");
System.err.println(e.toString());
System.exit(-100);
}
}
public String getAllPartitionInfo(String dbName){
List<String> res = Lists.newArrayList();
try {
List<String> tableList = hiveMetaStoreClient.getAllTables(dbName);
for(String tableName:tableList){
res.addAll(getTablePartitionInformation(dbName,tableName));
}
} catch (MetaException e) {
e.printStackTrace();
System.out.println("getAllTableStatistic error");
System.out.println(e.toString());
System.exit(-100);
}
return Joiner.on("\n").join(res);
}
public List<String> getTablePartitionInformation(String dbName, String tableName){
List<String> partitionsInfo = Lists.newArrayList();
try {
List<String> partitionNames = hiveMetaStoreClient.listPartitionNames(dbName,tableName, (short) 10000);
List<Partition> partitions = hiveMetaStoreClient.listPartitions(dbName,tableName, (short) 10000);
for(Partition partition:partitions){
StringBuffer sb = new StringBuffer();
sb.append(tableName);
sb.append("\t");
List<String> partitionValues = partition.getValues();
if(partitionValues.size()<4){
int size = partitionValues.size();
for(int j=0; j<4-size;j++){
partitionValues.add("null");
}
}
sb.append(Joiner.on("\t").join(partitionValues));
sb.append("\t");
DateTime createDate = new DateTime((long)partition.getCreateTime()*1000);
sb.append(createDate.toString("yyyy-MM-dd HH:mm:ss"));
partitionsInfo.add(sb.toString());
}
} catch (TException e) {
e.printStackTrace();
return Arrays.asList(new String[]{"error for request on" + tableName});
}
return partitionsInfo;
}
public String getAllTableStatistic(String dbName){
List<String> res = Lists.newArrayList();
try {
List<String> tableList = hiveMetaStoreClient.getAllTables(dbName);
for(String tableName:tableList){
res.addAll(getTableColumnsInformation(dbName,tableName));
}
} catch (MetaException e) {
e.printStackTrace();
System.out.println("getAllTableStatistic error");
System.out.println(e.toString());
System.exit(-100);
}
return Joiner.on("\n").join(res);
}
public List<String> getTableColumnsInformation(String dbName, String tableName){
try {
List<FieldSchema> fields = hiveMetaStoreClient.getFields(dbName, tableName);
List<String> infs = Lists.newArrayList();
int cnt = 0;
for(FieldSchema fs : fields){
StringBuffer sb = new StringBuffer();
sb.append(tableName);
sb.append("\t");
sb.append(cnt);
sb.append("\t");
cnt++;
sb.append(fs.getName());
sb.append("\t");
sb.append(fs.getType());
sb.append("\t");
sb.append(fs.getComment());
infs.add(sb.toString());
}
return infs;
} catch (TException e) {
e.printStackTrace();
System.out.println("getTableColumnsInformation error");
System.out.println(e.toString());
System.exit(-100);
return null;
}
}
}

If you want to have the ability to run such queries that return hive metadata, you can setup Hive metastore with MySQL, metadata used in Hive is stored in a specific account of MySQL.
You will have to create a user of MySQL for hive by doing CREATE USER 'hive'#'metastorehost' IDENTIFIED BY 'mypassword'.
Then you will find tables like COLUMNS_VS with the info you are looking for.
An example query to retrieve all columns in all tables could be: SELECT COLUMN_NAME, TBL_NAME FROM COLUMNS_V2 c JOIN TBLS a ON c.CD_ID=a.TBL_ID
Alternatively, you can access this information via REST calls to WebHCat see wiki for more info.

How to access Excel as Database using Java 8

I am using Java 8.
When i am trying to access Excel data(basically this is my test data) through jdbc-odbc, i am getting "java.lang.ClassNotFoundException: sun.jdbc.odbc.JdbcOdbcDriver"
And also i am trying to access data as non DSN.
I surfed net and came to know that Oracle deprecated support to jdbc-odbc.
So what is the easiest way to access this Excel data using Java?
Connection con=null;
Statement stmt=null;
ResultSet rs=null;
String query = "select TestScript from [TS 360 Scripts$]";
try
{
Class.forName( "sun.jdbc.odbc.JdbcOdbcDriver" );
con = DriverManager.getConnection("jdbc:odbc:;Driver={Microsoft Excel Driver(*.xlsx)};DBQ=D://TS 360 Script with Count.xlsx");
stmt=con.createStatement();
rs=stmt.executeQuery(query);
while(rs.next())
{
System.out.println(rs.getString("TestScript"));
}
con.close();
rs.close();
stmt.close();
}
catch(Exception e)
{
e.printStackTrace();
}

Uday- you can easily do whatever you want to do with Apache POI jar
As Your are mentioned your requirement: of all rows having isExecuted String Yes. I tried with this jar.
Try this
package com.dd.selenium;
import java.io.FileInputStream;
import java.io.IOException;
import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
public class PerformDDTest {
private static HSSFWorkbook xlWBook;
private static HSSFSheet xlSheet;
private static HSSFRow xlRow;
private static HSSFCell xlCell;
private static String filePath = "/home/dinesh/";
private static String fileName = "test.xls";
public static void main(String[] args) throws InterruptedException {
try {
FileInputStream xlFile = new FileInputStream(filePath + fileName);
// Access the required test data sheet
xlWBook = new HSSFWorkbook(xlFile);
// Assuming your data is in Sheet1- if not use your own sheet name
xlSheet = xlWBook.getSheet("Sheet1");
// gives row count in sheet
int noOfRows = xlSheet.getPhysicalNumberOfRows();
// gives column count in sheet
xlRow = xlSheet.getRow(0);
int noOfColumns = xlRow.getLastCellNum();
// excelData - 2 dimm array - stores all the excel data -Sheet1 only
String[][] excelData = new String[noOfRows][noOfColumns];
// r - row c- column
for (int r = 1; r < noOfRows; r++) {
for (int c = 0; c < noOfColumns; c++) {
xlRow = xlSheet.getRow(r);
xlCell = xlRow.getCell(c);
// Here we have complete excel data in an array -excelData-
excelData[r][c] = xlCell.getStringCellValue();
// System.out.println("row: " + r + " column: " + c);
// System.out.println(excelData[r][c]);
}
}
// creating an array to store isExected column
String[][] isExecuted = new String[noOfRows][1];
for (int row = 1; row < noOfRows; row++) {
// here column is always only one
// so c=0
// extracting a isExecuted column - and considering it as last
// column in sheet
// in your case it is not then - count the column position : use
// position-1
// ex: if column position is 7 then use 6 as below
// isExecuted[row][0]= excelData[row][6];
isExecuted[row][0] = excelData[row][noOfColumns - 1];
if (isExecuted[row][0].equalsIgnoreCase("yes")) {
// accessing complete row -which isExecuted=Yes
// *********IMPORTANT*****
for (int col = 0; col < noOfColumns; col++) {
// prints all the rows where isExecuted column has Yes
System.out.println(excelData[row][col]);
}
}
// System.out.println(isExecuted[row][0]);
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
I used this Excel Data:
Test Case Name Username Password Results IsExecute
APACHE_POI_TC testuser_1 Test#123 Pass Yes
APACHE_POI_TC testuser_2 Test#124 Pass No
APACHE_POI_TC testuser_3 Test#125 Pass Yes
APACHE_POI_TC testuser_4 Test#126 Pass Yes
APACHE_POI_TC testuser_5 Test#127 Pass No
APACHE_POI_TC testuser_6 Test#128 Pass Yes

Dont Access Excel file as a database. Instead use a jar such
as Apache POI For Microsoft Documents
Download Link: Apache POI For MS Docs- Jar
An Example For using this API:
Note: you must add apache poi jar to your build path before running it
package com.dd.selenium;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.openqa.selenium.By;
import org.openqa.selenium.firefox.FirefoxDriver;
public class PerformDDTest {
private static HSSFWorkbook xlWBook;
private static HSSFSheet xlSheet;
private static HSSFRow xlRow;
private static HSSFCell xlCell;
private static String filePath = "/home/dinesh/";
private static String fileName = "test.xls";
private static String url = "http://store.demoqa.com/";
private static String result = "Pass";
public static void main(String[] args) throws InterruptedException {
try {
FileInputStream xlFile =
new FileInputStream(filePath+fileName);
//Access the required test data sheet
xlWBook = new HSSFWorkbook(xlFile);
xlSheet = xlWBook.getSheet("Sheet1");
xlRow = xlSheet.getRow(1);
String username = xlRow.getCell(1).getStringCellValue();
String password = xlRow.getCell(2).getStringCellValue();
FirefoxDriver driver = new FirefoxDriver();
driver.manage().timeouts().implicitlyWait(10, TimeUnit.SECONDS);
driver.manage().window().maximize();
driver.get(url);
driver.findElement(By.xpath(".//*[#id='account']/a")).click();
driver.findElement(By.id("log")).sendKeys(username);
driver.findElement(By.id("pwd")).sendKeys(password);
driver.findElement(By.id("login")).click();
driver.findElement(By.xpath(".//*[#id='account_logout']/a")).click();
Thread.sleep(5000);
driver.quit();
setResultCell();
FileOutputStream fout = new FileOutputStream(filePath+fileName);
xlWBook.write(fout);
fout.flush();
fout.close();
} catch (IOException e) {
// TODO Auto-generated catch block
result = "Failed";
setResultCell();
e.printStackTrace();
}
}
private static void setResultCell() {
xlCell = xlRow.getCell(3, xlRow.RETURN_BLANK_AS_NULL);
if(xlCell == null ){
xlCell = xlRow.createCell(3);
xlCell.setCellValue(result);
}else{
xlCell.setCellValue(result);
}
}
}

This might be a little late but in case you still have the issue you can retain access excel as a db with java 8 using Fillo: http://codoid.com/fillo/

Read remote .csv file using opencsv

I've been pondering this for a fair amount of time now. I'm trying to download the data from Yahoo!'s Stock API. When you use the API, it gives you a .csv file. I've been looking at opencsv, which seems perfect, except I want to avoid downloading and saving the file, if at all possible.
OpenCSV, according to the examples, can only read from a FileReader. According to Oracle's docs on FileReader, the file needs to be local.
Is it possible to read from a remote file using OpenCSV without downloading?

CSVReader takes a Reader argument according to the documentation, so it isn't limited to a FileReader for the parameter.
To use a CSVReader without saving the file first, you could use a BufferedReader around a stream loading the data:
URL stockURL = new URL("http://example.com/stock.csv");
BufferedReader in = new BufferedReader(new InputStreamReader(stockURL.openStream()));
CSVReader reader = new CSVReader(in);
// use reader

Implementation of opencsv for reading csv file and saving to database.
import com.opencsv.CSVParser;
import com.opencsv.CSVParserBuilder;
import com.opencsv.CSVReader;
import com.opencsv.CSVReaderBuilder;
import com.opencsv.bean.CsvBindByPosition;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.persistence.Column;
import java.io.*;
import java.lang.annotation.Annotation;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
#Service
#Slf4j
public class FileUploadService {
#Autowired
private InsertCSVContentToDB csvContentToDB;
/**
* #param csvFileName location of the physical file.
* #param type Employee.class
* #param delimiter can be , | # etc
* #param obj //new Employee();
*
* import com.opencsv.bean.CsvBindByPosition;
* import lombok.Data;
*
* import javax.persistence.Column;
*
* #Data
* public class Employee {
*
* #CsvBindByPosition(position = 0, required = true)
* #Column(name = "EMPLOYEE_NAME")
* private String employeeName;
*
* #CsvBindByPosition(position = 1)
* #Column(name = "Employee_ADDRESS_1")
* private String employeeAddress1;
* }
*
* #param sqlQuery query to save data to DB
* #param noOfLineSkip make it 0(Zero) so that it should not skip any line.
* #param auditId apart from regular column in csv we need to add more column for traking like file id or audit id
* #return
*/
public <T> void readCSVContentInArray(String csvFileName, Class<? extends T> type, char delimiter, Object obj,
String sqlQuery, int noOfLineSkip, Long auditId) {
List<T> lstCsvContent = new ArrayList<>();
Reader reader = null;
CSVReader csv = null;
try {
reader = new BufferedReader(new InputStreamReader(new FileInputStream(csvFileName), "utf-8"));
log.info("Buffer Reader : " + ((BufferedReader) reader).readLine().isEmpty());
CSVParser parser = new CSVParserBuilder().withSeparator(delimiter).withIgnoreQuotations(true).build();
csv = new CSVReaderBuilder(reader).withSkipLines(noOfLineSkip).withCSVParser(parser).build();
String[] nextLine;
int size = 0;
int chunkSize = 10000;
Class params[] = { Long.class };
Object paramsObj[] = { auditId };
long rowNumber = 0;
Field field[] = type.getDeclaredFields();
while ((nextLine = csv.readNext()) != null) {
rowNumber++;
try {
obj = type.newInstance();
for (Field f : field) {
if(!f.isSynthetic()){
f.setAccessible(true);
Annotation ann[] = f.getDeclaredAnnotations();
CsvBindByPosition csv1 = (CsvBindByPosition) ann[0];
Column c = (Column)ann[1];
try {
if (csv1.position() < nextLine.length) {
if (csv1.required() && (nextLine[csv1.position()] == null
|| nextLine[csv1.position()].trim().isEmpty())) {
String message = "Mandatory field is missing in row: " + rowNumber;
log.info("null value in " + rowNumber + ", " + csv1.position());
System.out.println(message);
}
if (f.getType().equals(String.class)) {
f.set(obj, nextLine[csv1.position()]);
}
if (f.getType().equals(Boolean.class)) {
f.set(obj, nextLine[csv1.position()]);
}
if (f.getType().equals(Integer.class)) {
f.set(obj, Integer.parseInt(nextLine[csv1.position()]));
}
if (f.getType().equals(Long.class)) {
f.set(obj, Long.parseLong(nextLine[csv1.position()]));
}
if (f.getType().equals(Double.class) && null!=nextLine[csv1.position()] && !nextLine[csv1.position()].trim().isEmpty() ) {
f.set(obj, Double.parseDouble(nextLine[csv1.position()]));
}if(f.getType().equals(Double.class) && ((nextLine[csv1.position()]==null) || nextLine[csv1.position()].isEmpty())){
f.set(obj, new Double("0.0"));
}
if (f.getType().equals(Date.class)) {
f.set(obj, nextLine[csv1.position()]);
}
}
} catch (Exception fttEx) {
log.info("Exception when parsing the file: " + fttEx.getMessage());
System.out.println(fttEx.getMessage());
}
}
}
lstCsvContent.add((T) obj);
if (lstCsvContent.size() > chunkSize) {
size = size + lstCsvContent.size();
//write code to save to data base of file system in chunk.
lstCsvContent = null;
lstCsvContent = new ArrayList<>();
}
} catch (Exception ex) {
log.info("Exception: " + ex.getMessage());
}
}
//write code to save list into DB or file system
System.out.println(lstCsvContent);
} catch (Exception ex) {
log.info("Exception:::::::: " + ex.getMessage());
} finally {
try {
if (csv != null) {
csv.close();
}
if (reader != null) {
reader.close();
}
} catch (IOException ioe) {
log.info("Exception when closing the file: " + ioe.getMessage());
}
}
log.info("File Processed successfully: ");
}
}

Develop Reference

Java is a programming language and computing platform first released by Sun Microsystems in 1995.

Inserting records into Berkeley DB with Java - java

I can't seem to find sample code for constructing a Berkeley DB in Java and inserting records into it. Any samples? And I do not mean the Berkeley DB Java Edition either.

Related

How to do HBase range scan for Hexadecimal row key?

Unable to access method in arraylist class from other classes in java

Hive, how do I retrieve all the database's tables columns

How to access Excel as Database using Java 8

Read remote .csv file using opencsv

Categories

Resources