I have to pass record to an UDF which calls an API but as we want to do it parallely,we are using spark and thats why UDF is being developed, the problem here is that that UDF needs to take only 100 records at a time not more than that, it can't handle more than 100 records parallely, so how to ensure that only 100 record pass to it in one go please note we don't want to use count() function on whole record.
I am attaching the UDF code here,it's a generic UDF which returns array of struct.moreover if we pass 100 records in batchsize variable each time then,if suppose there are 198 records then if as we dont want to use count() we will not be knowing that its last batchsize is going to be 98.so how to handle that thing.
Guys... I have a generic UDF in which call is made for an API but before calling it creates batch of 100 firstly then only call restapi.. So the argument UDF takes are x1:string, x2:string, batchsize:integer(currently the batchsize is 100)..so in UDF until and unless the batchsize is not 100 the call will not happen.. And for each record it will return null.
So till 99th record it will return. Null but at 100th record the call will happen
[So, now the problem part:as we are taking batchsize 100 and call will take place only at 100th record. So, in condition like if we have suppose 198 record in file then 100 record will get the output but, other 98 will only return null as they will not get processed..
So please help a way around, and UDF take one record at a time, but it keep on collecting till 100th record.. I hope this clears up
public class Standardize_Address extends GenericUDF {
private static final Logger logger = LoggerFactory.getLogger(Standardize_Address.class);
private int counter = 0;
Client client = null;
private Batch batch = new Batch();
public Standardize_Address() {
client = new ClientBuilder().withUrl("https://ss-staging-public.beringmedia.com/street-address").build();
}
// StringObjectInspector streeti;
PrimitiveObjectInspector streeti;
PrimitiveObjectInspector cityi;
PrimitiveObjectInspector zipi;
PrimitiveObjectInspector statei;
PrimitiveObjectInspector batchsizei;
private ArrayList ret;
#Override
public String getDisplayString(String[] argument) {
return "My display string";
}
#Override
public ObjectInspector initialize(ObjectInspector[] args) throws UDFArgumentException {
System.out.println("under initialize");
if (args[0] == null) {
throw new UDFArgumentTypeException(0, "NO Street is mentioned");
}
if (args[1] == null) {
throw new UDFArgumentTypeException(0, "No Zip is mentioned");
}
if (args[2] == null) {
throw new UDFArgumentTypeException(0, "No city is mentioned");
}
if (args[3] == null) {
throw new UDFArgumentTypeException(0, "No State is mentioned");
}
if (args[4] == null) {
throw new UDFArgumentTypeException(0, "No batch size is mentioned");
}
/// streeti =args[0];
streeti = (PrimitiveObjectInspector)args[0];
// this.streetvalue = (StringObjectInspector) streeti;
cityi = (PrimitiveObjectInspector)args[1];
zipi = (PrimitiveObjectInspector)args[2];
statei = (PrimitiveObjectInspector)args[3];
batchsizei = (PrimitiveObjectInspector)args[4];
ret = new ArrayList();
ArrayList structFieldNames = new ArrayList();
ArrayList structFieldObjectInspectors = new ArrayList();
structFieldNames.add("Street");
structFieldNames.add("city");
structFieldNames.add("zip");
structFieldNames.add("state");
structFieldObjectInspectors.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector);
structFieldObjectInspectors.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector);
structFieldObjectInspectors.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector);
structFieldObjectInspectors.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector);
StructObjectInspector si2 = ObjectInspectorFactory.getStandardStructObjectInspector(structFieldNames,
structFieldObjectInspectors);
ListObjectInspector li2;
li2 = ObjectInspectorFactory.getStandardListObjectInspector(si2);
return li2;
}
#Override
public Object evaluate(DeferredObject[] args) throws HiveException {
ret.clear();
System.out.println("under evaluate");
// String street1 = streetvalue.getPrimitiveJavaObject(args[0].get());
Object oin = args[4].get();
System.out.println("under typecasting");
int batchsize = (Integer) batchsizei.getPrimitiveJavaObject(oin);
System.out.println("batchsize");
Object oin1 = args[0].get();
String street1 = (String) streeti.getPrimitiveJavaObject(oin1);
Object oin2 = args[1].get();
String zip1 = (String) zipi.getPrimitiveJavaObject(oin2);
Object oin3 = args[2].get();
String city1 = (String) cityi.getPrimitiveJavaObject(oin3);
Object oin4 = args[3].get();
String state1 = (String) statei.getPrimitiveJavaObject(oin4);
logger.info("address passed, street=" + street1 + ",zip=" + zip1 + ",city=" + city1 + ",state=" + state1);
counter++;
try {
System.out.println("under try");
Lookup lookup = new Lookup();
lookup.setStreet(street1);
lookup.setCity(city1);
lookup.setState(state1);
lookup.setZipCode(zip1);
lookup.setMaxCandidates(1);
batch.add(lookup);
} catch (BatchFullException ex) {
logger.error(ex.getMessage(), ex);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
/* batch.add(lookup); */
if (counter == batchsize) {
System.out.println("under if");
try {
logger.info("batch input street " + batch.get(0).getStreet());
try {
client.send(batch);
} catch (Exception e) {
logger.error(e.getMessage(), e);
logger.warn("skipping current batch, continuing with the next batch");
batch.clear();
counter = 0;
return null;
}
Vector<Lookup> lookups = batch.getAllLookups();
for (int i = 0; i < batch.size(); i++) {
// ListObjectInspector candidates;
ArrayList<Candidate> candidates = lookups.get(i).getResult();
if (candidates.isEmpty()) {
logger.warn("Address " + i + " is invalid.\n");
continue;
}
logger.info("Address " + i + " is valid. (There is at least one candidate)");
for (Candidate candidate : candidates) {
final Components components = candidate.getComponents();
final Metadata metadata = candidate.getMetadata();
logger.info("\nCandidate " + candidate.getCandidateIndex() + ":");
logger.info("Delivery line 1: " + candidate.getDeliveryLine1());
logger.info("Last line: " + candidate.getLastLine());
logger.info("ZIP Code: " + components.getZipCode() + "-" + components.getPlus4Code());
logger.info("County: " + metadata.getCountyName());
logger.info("Latitude: " + metadata.getLatitude());
logger.info("Longitude: " + metadata.getLongitude());
}
Object[] e;
e = new Object[4];
e[0] = new Text(candidates.get(i).getComponents().getStreetName());
e[1] = new Text(candidates.get(i).getComponents().getCityName());
e[2] = new Text(candidates.get(i).getComponents().getZipCode());
e[3] = new Text(candidates.get(i).getComponents().getState());
ret.add(e);
}
counter = 0;
batch.clear();
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
return ret;
} else {
return null;
}
}
}
Related
My problem is in the part where I'm doing the "if/else" conditions, when I call the function that will perform the comparisons and will define if the test passed or not and will send some information, I'm receiving null.
Problems are among the asterisks. If anyone can help me
This is my code :
public static void fxSpot_GBP_JPY(TradeData data, TradeData output) throws Exception {
if (data == null) {
fail("The input data object was not correctly filled");
}
if (output == null) {
fail("The output data object was not correctly filled");
}
//Used to set the comment, the status and update to JIRA
FieldsJSON fields = new FieldsJSON();
String assertionError = "";
List<String> inputs = new ArrayList<String>();
List<String> outputs = new ArrayList<String>();
String newDate = Utils.formatTimeZoneMinute(data.getTradeDate());
String asOfTrade = Utils.formatTimeZoneMinute(data.getAsOfTradeDate());
String executionDate = Utils.formatTimeZoneMinute(output.getExecutionDateTime());
try {
//Add the data in the list
inputs.add(data.getTransactionNumber()); outputs.add(output.getBloombergId());
inputs.add(newDate); outputs.add(output.getTradeDate());
inputs.add(asOfTrade); outputs.add(executionDate);
inputs.add(data.getSettlementDate()); outputs.add(output.getValueDate());
inputs.add(data.getTradeAmount()); outputs.add(output.getAmount2());
inputs.add(data.getCustomerAccountCounterparty()); outputs.add(output.getMiPartyId());
inputs.add(data.getPrincipalLoanAmount()); outputs.add(output.getAmount());
inputs.add(data.getSecurityPrice()); outputs.add(output.getRate());
inputs.add(data.getISOCodeOf1stCurrency()); outputs.add("BRL");//output.getCurrency2()
inputs.add(data.getISOCodeOf2ndCurrency()); outputs.add(output.getCurrency1());
//Compare values
System.out.println("-------------------");
int y = 0;
int x = 0;
for(String input : inputs) {
for(String out : outputs) {
if(y == x) {
if(input.equals(out)) {
WriterCSV.setOk("Ok");
**String comment = input + " = " + out;
fields.setComment(comment);
fields.setStatus("PASS");**
System.out.println("ok - " + input + " = " + out);
}else {
WriterCSV.setOk("not Ok");
**String comment = input + " = " + out;
fields.setComment(comment);
fields.setStatus("FAIL");**
System.out.println("not Ok - " + input + " = " + out);
}
}
x = x+1; // count of the list of output
}
y = y+1; // count of the list of inputs
x = 0; // reset to 0 the count of outputs
}
// evidence with the name and value of fields compared
WriterCSV.reportSpot_CSV(data,output);
}
Here is my test:
#Test
#Tag("compare")
public void CompareSpot() throws Exception {
//Create a list to read the CSVfile
List<DTOTradeData> dto;
//Used to get the TradeData of list dto.
DTOTradeData dtd = new DTOTradeData();
// Read a csvFile and return a list with the values to new xml
dto = CSVReader.readCSV("spot.csv");
//The xpath of xml
FileDriverSpot spot = new FileDriverSpot();
FileDriver output = new FileDriverSpotOutput();
FieldsJSON fields = new FieldsJSON();
//new xml = dataInput and the outputFile = dataOutput
TradeData dataInput = new TradeData();
TradeData dataOutput = new TradeData();
for (int i = 0; i < dto.size(); i++) {
dtd = dto.get(i); // get TradeData
dtd.getTradeData().setDriver(spot); // set the driver
if (fileExist(Setup.xmlPath + dtd.getInputFile() + ".xml")) {
dataInput = Reader.read(spot, Setup.xmlPath + dtd.getInputFile() + ".xml");
dataOutput = Reader.read(output, Setup.spotPath + dtd.getOutputFile());
try {
// make the comparison
**FunctionalTest.fxSpot_GBP_JPY(dataInput, dataOutput);**
}
catch(AssertionError e) {
String comment = e.toString();
fields.setComment(comment);
}
} else {
fail("The file: " + dtd.getTemplateFile()
+ " needs to go through the writing process before being compared.");
}
//Convert the file to base64
String inputData = UpdateTestStatus.convertToBase64(Setup.xmlPath + dtd.getInputFile() + ".xml");
String outputData = UpdateTestStatus.convertToBase64(Setup.spotPath + dtd.getOutputFile());
String evidenceCompared = UpdateTestStatus.convertToBase64(Setup.reportPath+"ReportSpot.csv");
System.out.println(UpdateTestStatus.updateTestRun(**fields.getStatus(),fields.getComment()**,
inputData,dtd.getInputFile()+ ".xml", //data of the XML and the name of the file
outputData,dtd.getOutputFile(),
evidenceCompared,"ReportSpot.csv",
Setup.testExec, dtd.getJiraId()).asString()); // ID testExecution and ID of
}
}
The test and the code under test each create a separate instance of FieldsJSON. Data set in one instance will not be visible in the other (unless the data is declared static, in which case there's no need to create instances).
You can fix this by using a single instance, either passed to the fxSpot_GBP_JPY method from the test, or returned from that method to the test.
Recently, I review the kafka code and test. I found a strange case:
I print the bytebuffer on the entry of SocketServer processCompletedReceives, as well as print the value on the point of Log sotre as follows:
the entry of SocketServer
private def processCompletedReceives() {
selector.completedReceives.asScala.foreach { receive =>
try {
openOrClosingChannel(receive.source) match {
case Some(channel) =>
val header = RequestHeader.parse(receive.payload)
val connectionId = receive.source
val context = new RequestContext(header, connectionId, channel.socketAddress,
channel.principal, listenerName, securityProtocol)
val req = new RequestChannel.Request(processor = id, context = context,
startTimeNanos = time.nanoseconds, memoryPool, receive.payload, requestChannel.metrics)
if(header.apiKey() == ApiKeys.PRODUCE){
LogHelper.log("produce request: %v" + java.util.Arrays.toString(receive.payload.array()))
}
...
the point of Log
validRecords.records().asScala.foreach { record =>
LogHelper.log("buffer info: value " + java.util.Arrays.toString(record.value().array()))
}
but, the result of print is different. and record.value() is not what I passed in client value like this:
public void run() {
int messageNo = 1;
while (true) {
String messageStr = "Message_" + messageNo;
long startTime = System.currentTimeMillis();
if (isAsync) { // Send asynchronously
producer.send(new ProducerRecord<>(topic,
messageNo,
messageStr), new DemoCallBack(startTime, messageNo, messageStr));
} else { // Send synchronously
try {
producer.send(new ProducerRecord<>(topic,
messageNo,
messageStr)).get();
System.out.println("Sent message: (" + messageNo + ", " + messageStr + ")");
} catch (InterruptedException | ExecutionException e) {
e.printStackTrace();
}
}
++messageNo;
}
}
the print result is not the not String messageStr = "Message_" + messageNo;
so what happend in the case.
done. I write the code as follows:
public class KVExtractor {
private static final Logger logger = LoggerFactory.getLogger(KVExtractor.class);
public static Map.Entry<byte[], byte[]> extract(Record record) {
if (record.hasKey() && record.hasValue()) {
byte[] key = new byte[record.key().limit()];
record.key().get(key);
byte[] value = new byte[record.value().limit()];
record.value().get(value);
System.out.println("key : " + new String(key) + " value: " + new String(value));
return new AbstractMap.SimpleEntry<byte[], byte[]>(key, value);
}else if(record.hasValue()){
// illegal impl
byte[] data = new byte[record.value().limit()];
record.value().get(data);
System.out.println("no key but with value : " + new String(data));
}
return null;
}
}
After successfully fetching alarms from Corba U2000 server and now reading the values, I am getting the error below
ERROR: org.omg.CORBA.MARSHAL: Sequence length too large. Only 12 available and trying to assign 31926513 vmcid: 0x0 minor code: 0 completed: No
org.omg.CORBA.MARSHAL: Sequence length too large. Only 12 available and trying to assign 31926513 vmcid: 0x0 minor code: 0 completed: No
at org.omg.CosNotification.EventBatchHelper.read(EventBatchHelper.java:57)
at AlarmIRPConstDefs.AlarmInformationSeqHelper.read(AlarmInformationSeqHelper.java:51)
at AlarmIRPConstDefs.AlarmInformationSeqHelper.extract(AlarmInformationSeqHelper.java:26)
at com.be.u2k.Main.getAlarmsList(Main.java:144)
at com.be.u2k.Main.main(Main.java:109)
for method AlarmInformationSeqHelper.extract
// Get all active alarms list
private static void getAlarmsList(ORB orb, AlarmIRP alarmIRP) {
try {
ManagedGenericIRPConstDefs.StringTypeOpt filter = new ManagedGenericIRPConstDefs.StringTypeOpt();
filter.value("($type_name == 'x1')"); // Query new alarms and acknowledge or unacknowledge alarms
AlarmIRPConstDefs.DNTypeOpt base_object = new AlarmIRPConstDefs.DNTypeOpt();
BooleanHolder flag = new BooleanHolder();
AlarmIRPSystem.AlarmInformationIteratorHolder iter = new AlarmIRPSystem.AlarmInformationIteratorHolder();
StructuredEvent[] alarmList = alarmIRP.get_alarm_list(filter, base_object, flag, iter);
System.out.println("AlarmIRP get_alarm_list success, flag: " + flag.value + " fetched total: " + (alarmList == null? -1: alarmList.length));
for (StructuredEvent alarm: alarmList) {
if (alarm.header != null) {
System.out.println("fixed_header.event_type.name: " + alarm.header.fixed_header.event_type.type_name
+ " fixed_header.event_type.domain_name: " + alarm.header.fixed_header.event_type.domain_name);
if (alarm.header.variable_header != null) {
for (Property variableHeader: alarm.header.variable_header) {
System.out.println("variable_header.name: " + variableHeader.name + " alarm.header.variable_header.value: " + variableHeader.value);
}
}
}
if (alarm.filterable_data != null) {
for (Property filterableData: alarm.filterable_data) {
System.out.println("data.name: " + filterableData.name);
if (filterableData.value != null && filterableData.value.toString().contains("org.jacorb.orb.CDROutputStream")) {
StructuredEvent[] filterableDataValues = AlarmInformationSeqHelper.extract(filterableData.value);
} else {
System.out.println("data.value: " + filterableData.value);
}
}
}
}
} catch (ManagedGenericIRPSystem.InvalidParameter e) {
System.out.println("ERROR get_alarm_list InvalidParameter (Indicates that the parameter is invalid): " + e) ;
} catch (ManagedGenericIRPSystem.ParameterNotSupported e) {
System.out.println("ERROR get_alarm_list ParameterNotSupported (Indicates that the operation is not supported): " + e) ;
} catch (AlarmIRPSystem.GetAlarmList e) {
System.out.println("ERROR get_alarm_list ParameterNotSupported (Indicates exceptions caused by unknown reasons): " + e) ;
}
}
Or is my way of reading the alarms list incorrect? Thanks.
You can find the example method below for getAlarmList
//Connect to AlarmIRP
AlarmIRP alarmIRP = AlarmIRPHelper.narrow(orb.string_to_object(alarmIrpIOR.value));
StringTypeOpt alarmFilter = new StringTypeOpt();
alarmFilter.value("");
DNTypeOpt base_object = new DNTypeOpt();
base_object.value("");
BooleanHolder flag = new BooleanHolder(false); // false for iteration
AlarmInformationIteratorHolder iter = new AlarmInformationIteratorHolder();
List<String> alarmIds = get_alarm_list(alarmIRP, alarmFilter, base_object, flag, iter);
private List<String> get_alarm_list(org._3gppsa5_2.AlarmIRPSystem.AlarmIRP alarmIRP, org._3gppsa5_2.ManagedGenericIRPConstDefs.StringTypeOpt alarmFilter, org._3gppsa5_2.AlarmIRPConstDefs.DNTypeOpt base_object, BooleanHolder flag, org._3gppsa5_2.AlarmIRPSystem.AlarmInformationIteratorHolder iter) throws org._3gppsa5_2.AlarmIRPSystem.GetAlarmList, org._3gppsa5_2.ManagedGenericIRPSystem.ParameterNotSupported, org._3gppsa5_2.AlarmIRPSystem.NextAlarmInformations, org._3gppsa5_2.ManagedGenericIRPSystem.InvalidParameter, BAD_OPERATION {
logger.info("[get-alarm-list][start]");
alarmIRP.get_alarm_list(alarmFilter, base_object, flag, iter);
List<StructuredEvent> alarms = new ArrayList();
EventBatchHolder alarmInformation = new EventBatchHolder();
short alarmSize = 100;
List<String> alarmIds = new ArrayList();
while (iter.value.next_alarmInformations(alarmSize, alarmInformation)) {
alarms.addAll(Arrays.asList(alarmInformation.value));
logger.info("Current alarm size:" + alarms.size());
}
for (StructuredEvent event : alarms) {
try {
//printAlarm(event);
} catch (Exception ex) {
}
List<Property> rem = new ArrayList<Property>();
rem.addAll(Arrays.asList(PropertySeqHelper.extract(event.remainder_of_body)));
for (Property property : rem) {
if (!property.name.equals(org._3gppsa5_2.AlarmIRPNotifications.NotifyNewAlarm.ALARM_ID)) {
continue;
}
alarmIds.add(property.value.extract_string());
}
}
logger.info("[get-alarm-list][completed] size :" + alarms.size());
return alarmIds;
}
I managed to figure out what is that filterableData.value.toString() value that is "org.jacorb.orb.CDROutputStream". It turns out that the property with name "b" is a TimeBase:: UtcT according to the docs.
To convert it to correct value which is a utc timestamp, I changed the condition to
if (filterableData.name.equals("b") && filterableData.value != null && filterableData.value.toString().contains("org.jacorb.orb.CDROutputStream")) {
long occuranceTime = TimeTHelper.read(filterableData.value.create_input_stream());
System.out.println("data.value: " + occuranceTime);
}
I am creating a query using JCO, SAP util for the following code for example:
public static void TEST() throws JCoException {
JCoDestination destination;
JCoRepository sapRepository;
destination = JCoDestinationManager.getDestination(ABAP_AS);
JCoDestinationManager.getDestination(ABAP_AS);
System.out.println("Attributes:");
System.out.println(destination.getAttributes());
System.out.println();
try {
JCoContext.begin(destination);
sapRepository = destination.getRepository();
if (sapRepository == null) {
System.out.println("Couldn't get repository!");
System.exit(0);
}
JCoFunctionTemplate functionTemplate = sapRepository.getFunctionTemplate("EM_GET_NUMBER_OF_ENTRIES");
JCoFunction function = functionTemplate.getFunction();
JCoTable itTable = function.getTableParameterList().getTable("IT_TABLES");
itTable.appendRow();
itTable.setValue("TABNAME", "USR02");
// JCoTable returnOptions_ = function.getTableParameterList().getTable("OPTIONS");
// returnOptions_.appendRow();
//// //returnOptions.setValue("TEXT", "MODDA GE '20140908' AND MODTI GT '000000'");
// returnOptions_.setValue("TEXT", "BNAME EQ 'USER'");
function.execute(destination);
System.out.println( function.getTableParameterList().getTable("IT_TABLES").getInt("TABROWS"));
JCoFunctionTemplate template2 = sapRepository.getFunctionTemplate("RFC_READ_TABLE");
System.out.println("Getting template");
JCoFunction function2 = template2.getFunction();
function2.getImportParameterList().setValue("QUERY_TABLE", "USR02");
function2.getImportParameterList().setValue("DELIMITER", ",");
function2.getImportParameterList().setValue( "ROWCOUNT",5);
function2.getImportParameterList().setValue( "ROWSKIPS",5);
System.out.println("Setting OPTIONS");
// Date date = new Date(1410152400000L);
SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss");
// String dateString = formatter.format(date);
// String dt = dateString.substring(0, 8);
// String tm = dateString.substring(8);
// System.out.println("dt > " + dt + ", tm > " + tm);
JCoTable returnOptions = function2.getTableParameterList().getTable("OPTIONS");
returnOptions.appendRow();
//returnOptions.setValue("TEXT", "MODDA GE '20140908' AND MODTI GT '000000'");
returnOptions.setValue("TEXT", "BNAME LIKE 'S%'");
// returnOptions.appendRow();
// returnOptions.setValue("TEXT", "AND TYPE = 'DN'");
System.out.println("Setting FIELDS");
JCoTable returnFields = function2.getTableParameterList().getTable("FIELDS");
returnFields.appendRow();
returnFields.setValue("FIELDNAME", "BNAME");
returnFields.appendRow();
returnFields.setValue("FIELDNAME", "GLTGB");
returnFields.appendRow();
returnFields.setValue("FIELDNAME", "CLASS");
// returnFields.appendRow();
function2.execute(destination);
// JCoTable jcoTablef = function2.getTableParameterList().getTable("FIELDS");
JCoTable jcoTabled = function2.getTableParameterList().getTable("DATA");
int icodeOffSet = 0;
int icodeLength = 0;
int numRows = jcoTabled.getNumRows();
System.out.println("numRows > " + numRows);
for(int i=0; i<numRows; i++) {
jcoTabled.setRow(i);
System.out.println(jcoTabled.getRow());
String BNAME = "BNAE:" + jcoTabled.getString(0);
// String GLTGB = "GLTGB:" + jcoTabled.getString(2);
// String cls = "GLTGB:" + jcoTabled.getString(3);
System.out.println(BNAME + "..." );
}
} catch (Exception e) {
e.printStackTrace();
System.out.println("ERROR: " + e.getMessage());
} finally {
JCoContext.end(destination);
}
}
static void createDestinationDataFile(String destinationName, Properties connectProperties)
{
File destCfg = new File(destinationName+".jcoDestination");
try
{
FileOutputStream fos = new FileOutputStream(destCfg, false);
connectProperties.store(fos, "for tests only !");
fos.close();
}
catch (Exception e)
{
throw new RuntimeException("Unable to create the destination files", e);
}
}
The previous code worked well when I used the EQ operator.
However, when I used the IN operator:
BNAME IN ('USER1','USER','USER3')
or
BNAME EQ 'USER1' OR BNAME EQ 'USER' OR BNAME EQ 'USER3'
It throws an exception: Unexpected dynamic condition
Are there any limitations to the condition size? Since I have 22 field in the IN condition and each value has a size of 10?
You need to specify a valid OpenSQL condition, you need to observe the rules for dynamic conditions and you need to ensure that the condition is properly split into lines of 72 characters. My guess would be that the last bit might have been an issue if you're specifying 22 conditions...
I have a complicated situation here,..
I am developing an multithreaded application There are 5 threads which i am creating and using these threads i want to write a program to write into 5 files simultaneously and also limiting the number of data to 20000. I am getting data from DB. The data is different for all the 5 threads. So the correct data should go into each file. Also, each thread also needs to access correct filewriter inorder to write correct data in correct file and have counter for each file.
public class AuditMissingRelationshipGuidEntitiesToXml
{
private Logger logger = ERDLoggerUtility.getLogger( this.getClass().getName());
private DbPartitionedConnections myDbPConn = null;
private RelationshipRepositoryDao myRRDao = null;
private MrdPartitionedConnections myMRDPartitionConn = null;
private MasterRecordDao myMRDRDao = null;
private String myCollectionName = null;
private String myOutputDir = null;
private String myOutputFileName = null;
private EntityFileWriter myEntityFileWriter = null;
private NovusDocumentGuidCaseFinder novusDocGuidCaseFinder = null;
//private int entityFileWriterCount = 0;
private static List <String> collections = new ArrayList<String>();
private static final String DBDRIVER = "oracle.jdbc.driver.OracleDriver";
private BasicDataSource ds;
private static final int NTHREADS = 30;
private void setupDatabases() throws Exception {
ErdProperties props = ErdProperties.getProps();
this.myDbPConn = new RrdPartitionedConnections();
this.myRRDao = new OracleRelationshipRepositoryDao(this.myDbPConn);
this.myMRDPartitionConn = new MrdPartitionedConnections();
this.myMRDRDao = new OracleMasterRecordDao(this.myMRDPartitionConn);
ds = new BasicDataSource();
ds.setDriverClassName(DBDRIVER);
ds.setUrl(props.getProperty("wfdb.uri"));
ds.setUsername(props.getProperty("wfdb.user"));
ds.setPassword(props.getProperty("wfdb.password"));
novusDocGuidCaseFinder = new NovusDocumentGuidCaseFinder(ds);
novusDocGuidCaseFinder.setNovusCollection(myCollectionName);
}
private void setCommandLineProperties(String[] theArgs) throws Exception {
if (theArgs == null || theArgs.length != 3 ||
theArgs[0] == null || theArgs[0].length() == 0 ||
theArgs[1] == null || theArgs[1].length() == 0 ||
theArgs[2] == null || theArgs[2].length() == 0) {
System.err.println("You must provide 3 arguments! See below.");
System.err.println("\t1st Argument) -- Output directory location. ex: \"/ct-data/RRDGuids\"");
System.err.println("\t2nd Argument) -- Output file name. ex: \"rrd_seq_guid\"");
System.err.println("\t3rd Argument) -- Collection name. ex: \"N_ARREST\"");
throw new Exception("Invalid Arguments, see console output.");
}
else {
// Set the output directory location
this.myOutputDir = theArgs[0];
System.out.println("setCommandLineProperties() -- 1st Argument) Input directory location = [" + this.myOutputDir + "]");
// Set the output file name
this.myOutputFileName = theArgs[1];
System.out.println("setCommandLineProperties() -- 1st Argument) Input file name = [" + this.myOutputFileName + "]");
// Set the output file name
//this.myCollectionName = theArgs[2];
//System.out.println("setCommandLineProperties() -- 3nd Argument) Collection name = [" + this.myCollectionName + "]");
}
}
/**
* Add the rel seq and entity guid.
*
* #param theRelSeq
* #param theEntityGuid
* #throws Exception
*/
private synchronized void addEntity(String theRelSeq, String theEntityGuid,
String theRelGuid, EntityFileWriter myEntityFileWriter) throws Exception {
/*
if (this.myEntityFileWriter == null) {
System.out.println("file for :"+collectionName);
this.myEntityFileWriter = new EntityFileWriter(new File(myOutputDir), myOutputFileName, collectionName, System.currentTimeMillis());
}
*/
myEntityFileWriter.addEntity(theRelSeq, theEntityGuid, theRelGuid);
}
private Set<String> locateEntityGuidsInMRD(List<String> theRRDGuids, String collectionName) throws Exception {
Set<String> foundEntityGuids = new HashSet<String>();
System.out.println("Searching MRD for entity guids (AKA: BASE_GUID)... for : " + collectionName);
MasterRecordDao myMRDRDao = null;
synchronized(collectionName.intern()){
myMRDRDao = new OracleMasterRecordDao(this.myMRDPartitionConn);
/*for(String li : theRRDGuids){
System.out.println(li);
}*/
List<PersonEntity> personEntities = myMRDRDao.getByEntityGuid(theRRDGuids);
System.out.println("Found: " + (personEntities == null ? "0" : personEntities.size()) + " entities in MRD for :" + collectionName);
if (personEntities != null) {
for (PersonEntity pe : personEntities) {
foundEntityGuids.add(pe.getEntityGuid());
}
}
this.notify();
}
myMRDRDao=null;
return foundEntityGuids;
}
public void determineOrphanRelationships(String collectionName) throws Exception {
Long startTime = null;
Long endTime = null;
startTime = System.currentTimeMillis();
System.out.println("Started on: " + new Timestamp(startTime).toString() + "::" + collectionName);
synchronized(collectionName.intern()){
EntityFileWriter myEntityFileWriter = null;
Map<EntityDocRelationship, String> entityDocRels = new HashMap<EntityDocRelationship, String>();
EntityDocRelationshipIterator entityDocRelIter = myRRDao.getByCollection(collectionName);
int entityFileWriterCount = 0;
while (entityDocRelIter.hasNext()) {
/*
* Reference: OracleRelationshipRepositoryDao.populateFrom()
*
* EntityDocRelationship.getPk() = EntityDocRelationship.REL_SEQ (1st column in prepared statement)
* EntityDocRelationship.getRelGuid() = EntityDocRelationship.REL_GUID (4th position in prepared statement)
* EntityDocRelationship.getEntityGuid() = EntityDocRelationship.BASE_GUID (5th position in prepared statement)
* EntityDocRelationship.getDocGuid() = EntityDocRelationship.TARGET_GUID (6th position in prepared statement)
*/
if(entityDocRels.size() < 20000) {
EntityDocRelationship entityDocRel = (EntityDocRelationship)entityDocRelIter.next();
entityDocRels.put(entityDocRel, entityDocRel.getEntityGuid());
}
else{
entityFileWriterCount = printNotFoundEntityGuids(entityDocRels, collectionName,
entityFileWriterCount, myEntityFileWriter, false);
}
this.wait();
}
if(entityDocRels.size() > 0){
entityFileWriterCount = printNotFoundEntityGuids(entityDocRels, collectionName,
entityFileWriterCount, myEntityFileWriter, true);
endTime = System.currentTimeMillis();
entityFileWriterCount = 0;
System.out.println("Ended on:" + endTime);
System.out.println("Run time: " + (endTime - startTime));
}
}
}
private int printNotFoundEntityGuids(Map<EntityDocRelationship, String> entityDocRels, String collectionName,
int entityFileWriterCount, EntityFileWriter myEntityFileWriter, boolean closeWriter) throws Exception{
// Do work with the distinct set of entity guids.
Set<String> foundEntityGuids = locateEntityGuidsInMRD(new ArrayList<String>(entityDocRels.values()), collectionName);
System.out.println(foundEntityGuids.size());
System.out.println("Printing not found entity guid list...");
for (Map.Entry<EntityDocRelationship, String> entry : entityDocRels.entrySet()) {
if (!foundEntityGuids.contains(entry.getValue())) {
synchronized(this){
if (entityFileWriterCount == 0) {
System.out.println("file for :"+collectionName);
myEntityFileWriter = new EntityFileWriter(new File(myOutputDir), myOutputFileName,
collectionName, System.currentTimeMillis());
}
entityFileWriterCount++;
addEntity(String.valueOf(entry.getKey().getPk()), entry.getValue(), entry.getKey().getRelGuid(), myEntityFileWriter);
System.out.println(collectionName + " :: Relationship pk: " + entry.getKey().getPk() + " is orphan to Entity: " + entry.getValue());
System.out.println("Finished printing.");
if(closeWriter || entityFileWriterCount >= 10){
System.out.println("Closing file writer...");
if (this.myEntityFileWriter != null) {
this.myEntityFileWriter.endDocument();
this.myEntityFileWriter=null;
}
System.out.println("File writer closed.");
}
}
}
}
return entityFileWriterCount;
}
public void deleteRelationships(final String collectionName) throws Exception {
SAXParserFactory factory = SAXParserFactory.newInstance();
SAXParser saxParser = factory.newSAXParser();
File outputDir = new File(this.myOutputDir);
FilenameFilter filter = new FilenameFilter() {
public boolean accept(File dir, String name) {
return name.startsWith("rrd_seq_guid_"+collectionName);
}
};
String[] files = outputDir.list(filter);
for (String file : files) {
logger.info("Getting relationships from file: " + file + "...");
EntityFileHandler parsedFile = new EntityFileHandler();
saxParser.parse(this.myOutputDir + file, parsedFile);
Map<String, String> relSeqsGuids = new HashMap<String, String>();
for (EntityFileHandler.EntityFile entityFile : parsedFile.getDocuments()){
relSeqsGuids.put(entityFile.getRelSeq(), entityFile.getRelGuid());
}
/*logger.info("Deleting relationship sequence guids...");
myRRDao.deleteRelSeqGuids(relSeqsGuids.keySet());
logger.info("Relationship sequence guids deleted.");*/
/*
logger.info("Validating relationship guids in Novus...");
List<String> validNovusRelGuids = novusDocGuidCaseFinder.search(
relSeqsGuids.values().toArray(new String[relSeqsGuids.values().size()]));
logger.info("Relationship guids are valid in Novus.");*/
List<String> validNovusRelGuids = new ArrayList<String>();
validNovusRelGuids.addAll(relSeqsGuids.values());
logger.info("Sending delete files to Novus...");
sendDeleteFileToNovus(file, validNovusRelGuids, collectionName);
logger.info("Delete files sent.");
}
}
/**
* This method will send delete files to Novus based on the list
* of relationship Guids passed in.
*
* Example delete file:
*
* <deletes>
* <collection>
* <name>...</name>
* <guid></guid>
* .
* . N times
* .
* <guid></guid>
* </collection>
* </deletes>
*
* #param theRelGuids - ERD Relationship Guids to send a delete file for.
* #throws Exception
*/
public void sendDeleteFileToNovus(String theFile, List<String> theRelGuids, String collectionName) {
logger.info("Sending delete files to Novus...");
try{
Document document = new DocumentImpl();
// Create the Delete Element (aka: root): <Deletes>
Element deleteElement = document.createElement("deletes");
/*
* Create the Collection Element <Collection> and then add it to the Deletes Element.
*
* <deletes>
* <collection></collection>
* </deletes>
*/
Element collectionElement = document.createElement("collection");
deleteElement.appendChild(collectionElement);
/*
* Create the Name Element <Name> with the Collection name
* and then add it to the Collection Element.
*
* <deletes>
* <collection>
* <name>...</Name>
* </collection>
* </deletes>
*/
Element nameElement = document.createElementNS(null, "name");
nameElement.appendChild(document.createTextNode(collectionName));
collectionElement.appendChild(nameElement);
if (theRelGuids != null && theRelGuids.size() > 0) {
/*
* For each Relationship Guid add a Guid Element to the Collection
* Element with the Relationship Guid.
*
* <deletes>
* <collection>
* <name>...</name>
* <guid>...</guid>
* </collection>
* </deletes>
*/
for(String relGuid : theRelGuids){
Element relGuidElement = document.createElementNS(null, "guid");
relGuidElement.appendChild(document.createTextNode(relGuid));
collectionElement.appendChild(relGuidElement);
}
}
logger.info("Add delete element text content: " + deleteElement.getTextContent());
document.appendChild(deleteElement);
String deleteFile =
File.separator + "ct-data" +
File.separator + "erd" +
File.separator + "wip" +
File.separator + "relAtishDeletes" +
File.separator + theFile;
logger.info("Creating Novus delete file: " + deleteFile + "...");
FileOutputStream outStream = new FileOutputStream(deleteFile);
OutputFormat of = new OutputFormat("XML","ISO-8859-1",true);
of.setIndent(1);
of.setIndenting(true);
XMLSerializer serializer = new XMLSerializer(outStream, of);
serializer.asDOMSerializer();
serializer.serialize(document.getDocumentElement());
outStream.close();
logger.info("Delete file: " + deleteFile + " created.");
}
catch(Exception e){
e.printStackTrace();
}
/*MatchManageCommand mmc = new MatchManageCommand();
String publishXML = mmc.getPublishXml(-1, this.myCollectionName, "-1", theFile);
mmc.ms.putMessageOnQueue(publishXML, 2);*/
}
public void getCollections()
{
Connection connection = null;
Statement statement = null;
ResultSet resultSet = null;
try {
connection = ds.getConnection();
String query = "select novus_collection from erd_workflow.collection_info " +
"where Novus_collection in ('N-SALE', 'N-UCC00', 'N-UCC01', 'N-UCC02', 'N-UCC03') order by novus_collection asc";
statement = connection.createStatement();
resultSet = statement.executeQuery(query);
while (resultSet.next()) {
// Store the Novus Collection and associated Product.
collections.add(resultSet.getString(1));
}
}
catch (SQLException sqle) {
sqle.printStackTrace();
} catch(Exception e){
e.printStackTrace();
}
finally {
if (resultSet != null) {
try {
resultSet.close();
statement.close();
connection.close();
} catch (SQLException sqle) {
sqle.printStackTrace();
}
}
}
}
public static void main(String[] args) throws Exception{
AuditMissingRelationshipGuidEntitiesToXml audit = new AuditMissingRelationshipGuidEntitiesToXml();
ExecutorService executor = Executors.newFixedThreadPool(NTHREADS); // makes thread pool
try {
audit.setupDatabases();
audit.getCollections();
audit.setCommandLineProperties(args);
//audit.runThreads(0, collections.size());
int start = 0;
while(start < collections.size()){
Runnable worker = audit.new RunThreads(audit, collections.get(start) );
/*
* Executes the given command at some time in the future. The command
* may execute in a new thread, in a pooled thread, or in the calling
* thread*/
executor.execute(worker);
start++; // increment to get next collection
}
/*
* Initiates an orderly shutdown in which previously submitted
* tasks are executed, but no new tasks will be
* accepted. Invocation has no additional effect if already shut
* down. */
executor.shutdown();
System.out.println("Finished all threads");
// System.out.println("done with current threads..... creating new threads... ");
}catch(InterruptedException e) {
System.out.println("Interrupted :");
e.printStackTrace();
}catch(RejectedExecutionException e){
e.printStackTrace();
}
}
public String getMyCollectionName() {
return myCollectionName;
}
public void setMyCollectionName(String myCollectionName) {
this.myCollectionName = myCollectionName;
}
public class RunThreads implements Runnable{
AuditMissingRelationshipGuidEntitiesToXml auditThread;
private String myCollection;
private int sleepTime; // random sleep time for thread
private Random generator = new Random();
public RunThreads(AuditMissingRelationshipGuidEntitiesToXml audit, String collection){
try{
this.auditThread = audit;
myCollection = collection;
// pick random sleep time between 0 and 5 seconds
sleepTime = generator.nextInt(20000);
}catch(Exception e){
System.out.println("Interrupted");
}
}
/* Synchronize the AuditMissingRelationshipGuidEntitiesToXml object and call the methods on the particular object */
public void run() {
synchronized(this.myCollection.intern()) { // synchronized block
try {
//Thread.sleep(sleepTime);
System.out.println("Collection started : "+ this.myCollection);
this.auditThread.determineOrphanRelationships(this.myCollection);
this.auditThread.deleteRelationships(this.myCollection);
} catch (Exception e) {
e.printStackTrace();
}
}
}
}