Develop Reference

How do I grab html data from inside a table?

I am trying to create a web scraper program that takes tables from a website and converts them into ".csv" files.
I'm using Jsoup to pull the data down into a document and have it read from document.html() doc.html() below. The reader as it stands picks up 18 tables at my test site but no table data tags.
Do you have any idea what could be going wrong?
ArrayList<Data_Log> container = new ArrayList<Data_Log>();
ArrayList<ListData_Log> containerList = new ArrayList<ListData_Log>();
ArrayList<String> tableNames = new ArrayList<String>();// Stores native names of tables
ArrayList<Double> meanStorage = new ArrayList<Double>();// Stores data mean per table
ArrayList<String> processlog = new ArrayList<String>();// Keeps a record of all actions taken per iteration
ArrayList<Double> modeStorage = new ArrayList<Double>();
Calendar cal;
private static final long serialVersionUID = -8174362940798098542L;
public void takeData() throws IOException {
if (testModeActive == true) {
System.out.println("Initializing Data Cruncher with developer logs");
System.out.println("Taking data from: " + dataSource); }
int irow = 0;
int icolumn = 0;
int iTable = 0;
// int iListno = 0;
// int iListLevel;
String u = null;
boolean recording = false;
boolean duplicate = false;
Document doc = Jsoup.connect(dataSource).get();
Webtitle = doc.title();
Pattern tb = Pattern.compile("<table");
Matcher tB = tb.matcher(doc.html());
Pattern ttl = Pattern.compile("<title>(//s+)</title>");
Matcher ttl2= ttl.matcher(doc.html());
Pattern tr = Pattern.compile("<tr");
Matcher tR = tr.matcher(doc.html());
Pattern td = Pattern.compile("<td(//s+)</td>");
Matcher tD = td.matcher(doc.html());
Pattern tdc = Pattern.compile("<td class=(//s+)>(//s+)</td>");
Matcher tDC = tdc.matcher(doc.html());
Pattern tb2 = Pattern.compile("</table>");
Matcher tB2 = tb2.matcher(doc.html());
Pattern th = Pattern.compile("<th");
Matcher tH = th.matcher(doc.html());
while (tB.find()) {
iTable++;
while(ttl2.find()) {
tableNames.add(ttl2.group(1));
}
while (tR.find()) {
while (tD.find()||tH.find()) {
u = tD.group(1);
Data_Log v = new Data_Log();
v.setTable(iTable);
v.dataSort(u);
v.setRow(irow);
v.setColumn(icolumn);
container.add(v);
icolumn++;
}
while(tDC.find()) {
u = tDC.group(2);
Data_Log v = new Data_Log();
v.setTable(iTable);
v.dataSort(u);
v.setRow(irow);
v.setColumn(icolumn);
container.add(v);
icolumn++;
}
irow++;
}
if (tB2.find()) {
irow=0;
icolumn=0;
}
}
Expected results:
table# logged + "td"s logged
Actual result:
table# logged "td"s omitted

Since you're using jsoup, use it
var url = "<your url>";
var doc = Jsoup.connect(url).get();
var tables = doc.body().getElementsByTag("table");
tables.forEach(table -> {
System.out.println(table.id());
System.out.println(table.className());
System.out.println(table.getElementsByTag("td"));
});
For your tries to parse html with regex, here's some suggested reading
Using regular expressions to parse HTML: why not?
Why is it such a bad idea to parse XML with regex?
RegEx match open tags except XHTML self-contained tags

Android Json Parsing with multipartEntity

I need to parse this as an JsonArray instead of objects
but it dosent seem to work correctly.
my output is ["{"TransactionVolume":"34","TransactionitemID":"2"}"]
I need it to be [{"TransactionVolume":"34","TransactionitemID":"2"}]
this is part of my code
public class TransactionItem {
public String TransactionitemID;
public String TransactionVolume;
public String getTransactionVolume() {
return TransactionVolume;
}
public void setTransactionVolume(String transactionVolume) {
TransactionVolume = transactionVolume;
}
public String getTransactionitemID() {
return TransactionitemID;
}
public void setTransactionitemID(String transactionitemID) {
TransactionitemID = transactionitemID;
}
}
TransactionItem transactionItem = new TransactionItem();
transactionItem.setTransactionitemID(article.toString());
transactionItem.setTransactionVolume(volume.toString());
String transitemjson = gson.toJson(transactionItem);
JSONArray ja = new JSONArray();
ja.put(transitemjson);
String test = ja.toString().replaceAll("\\\\\"", "\"");
Ja seems to be right where the output is
["{\"TransactionVolume\":\"34\",\"TransactionitemID\":\"2\"}"]
So I've tried to replace all \ with " but still the " at the begging and end.
Trimming the String doesn't seem like an good idea, but it could work. Is there any other way of doing this?
EDIT
This got me to the goal!
ArrayList<TransactionItem> Transactionlist = new ArrayList<>();
for (int i=0; i < CompleteArticle.size(); i++) {
String id = CompleteArticle.get(i);
String Volume = CompleteVolume.get(i);
TransactionItem transactionItem = new TransactionItem();
transactionItem.setTransactionitemID(id);
transactionItem.setTransactionVolume(Volume);
//transactionitems.transactionitems.add(transactionItem);
Transactionlist.add(transactionItem);
}
JsonElement transitemjson = gson.toJsonTree(transactionItems);
Output :
[{"TransactionVolume":"1","TransactionitemID":"5"},{"TransactionVolume":"3","TransactionitemID":"3"}]

You need to use a JSONElement instead of a String.
Replace this line:
String transitemjson = gson.toJson(transactionItem);
With this:
JSONElement transitemjson = gson.toJsonTree(transactionItem);

What is a cleaner, more elegant way to initialize all of these?

I am naming all of these one by one. Is there a method that takes less space?
public class Matt{
PImage matt,
imgLS1, imgLS2, imgLS3, imgRS1, imgRS2, imgRS3,
imgLSB1, imgLSB2, imgLSB3, imgRSB1, imgRSB2, imgRSB3,
imgLW1, imgLW2, imgLW3, imgRW1, imgRW2, imgRW3,
imgLWB1, imgLWB2, imgLWB3, imgRWB1, imgRWB2, imgRWB3;
public Matt(){
imgLS1 = loadImage("./Images/Matt/MattLS1.png");
imgLS2 = loadImage("./Images/Matt/MattLS2.png");
imgLS3 = loadImage("./Images/Matt/MattLS3.png");
imgRS1 = loadImage("./Images/Matt/MattRS1.png");
imgRS2 = loadImage("./Images/Matt/MattRS2.png");
imgRS3 = loadImage("./Images/Matt/MattRS3.png");
imgLSB1 = loadImage("./Images/Matt/MattLSB1.png");
imgLSB2 = loadImage("./Images/Matt/MattLSB2.png");
imgLSB3 = loadImage("./Images/Matt/MattLSB3.png");
imgRSB1 = loadImage("./Images/Matt/MattRSB1.png");
imgRSB2 = loadImage("./Images/Matt/MattRSB2.png");
imgRSB3 = loadImage("./Images/Matt/MattRSB3.png");
imgLW1 = loadImage("./Images/Matt/MattLW1.png");
imgLW2 = loadImage("./Images/Matt/MattLW2.png");
imgLW3 = loadImage("./Images/Matt/MattLW3.png");
imgRW1 = loadImage("./Images/Matt/MattRW1.png");
imgRW2 = loadImage("./Images/Matt/MattRW2.png");
imgRW3 = loadImage("./Images/Matt/MattRW3.png");
imgLWB1 = loadImage("./Images/Matt/MattLWB1.png");
imgLWB2 = loadImage("./Images/Matt/MattLWB2.png");
imgLWB3 = loadImage("./Images/Matt/MattLWB3.png");
imgRWB1 = loadImage("./Images/Matt/MattRWB1.png");
imgRWB2 = loadImage("./Images/Matt/MattRWB2.png");
imgRWB3 = loadImage("./Images/Matt/MattRWB3.png");
}
}

Put your images in a Map<String,PImage>, organizing the map by image suffix. As far as accessing the images is concerned, this approach may be slightly less convenient/efficient than using variables directly, but it will save you a lot of space:
static final String[] suffixes = new String[] {"LS1", "LS2", "LS3", ..., "RWB3"};
Map<String,PImage> images = new HashMap<String,PImage>();
public Matt() {
for (String suffix : suffixes) {
PImage image = loadImage("./Images/Matt/Matt"+suffix+".png");
images.put(suffix, image);
}
}

Since the "LS", etc., seem to have semantic meaning, I'd suggest a variation of the solution by #dasblinkenlight that uses an enum:
final int N_FILES = 3; // files/position -- could also be a variable
enum Position {
LS, RS, LSB, RSB, LW, RW, LWB, LRB
}
Map<Position, String[]> files = new EnumMap<>(Position.class);
for (Position pos : Position.values()) {
String[] posFiles = new String[N_FILES];
files.put(pos, posFiles);
for (int i = 1; i <= N_FILES; ++i) {
posFiles[i-1] = "./Images/Matt/Matt" + pos.name() + i + ".png";
}
}
Then you can access any element with code like this:
Position p = RS; // or any other value
int index = 0; // 0..(N_FILES-1), corresponding to suffixes 1..N_FILES
String fileName = files.get(p)[i];

How to add a test step in HP QC from Java

I am trying to access and modify test cases in HP QC by JAVA. The code is running successfully but the Step, status, Exec dates are not being updated.
Here is my code
for (Com4jObject obj : testInstances)
{
ITSTest testInstance = obj.queryInterface(ITSTest.class);
ITSTest tstest = obj.queryInterface(ITSTest.class);
IRunFactory runfactory = tstest.runFactory().queryInterface(IRunFactory.class);
IRun run=runfactory.addItem("RunNew").queryInterface(IRun.class);
Com4jObject step = run.stepFactory();
// run.field("Step #", "Step1");
run.status("Passed");
// Com4jObject steps = run.stepFactory();
// System.out.println(run.field("Actual Result"));
// run.field("Actual Result", "As Expected. Please find attachment with TC001");
if(tstest.name().contains("[1]TC001"))
{
try {
String fileName = new File(files.get(i)).getName();
String folderName = new File(files.get(i)).getParent();
System.out.println("FILE: "+fileName);
System.out.println("FOLDER: "+folderName);
IAttachmentFactory attachfac = tstest.attachments().queryInterface(IAttachmentFactory.class);
IAttachment attach = attachfac.addItem(fileName).queryInterface(IAttachment.class);
IExtendedStorage extAttach = attach.attachmentStorage().queryInterface(IExtendedStorage.class);
extAttach.clientPath(folderName);
extAttach.save(fileName, true);
//attach.description(Actual);
attach.post();
attach.refresh();
} catch(Exception e) {
System.out.println("QC Exceptione : "+e.getMessage());
}
}
run.post();
//AppLog.info(" Test Instance: %s", testInstance.name());
System.out.println(("Test Instance: %s"+ testInstance.name()));
}

You need to call post() on each item separately: Test run and each created step.
Example in C# starting from the point where you retrieve the step factory.
// Create test run
var oRunInstance = (QcClient.RunFactory)oTsTest.RunFactory;
var oRun = (QcClient.Run)oRunInstance.AddItem("Performance Test");
oRun.Status = "Passed";
oRun.Post();
oRun.Refresh();
// Create test run steps
var oTest = (QcClient.Test)oTsTest.Test;
var tsDesignStepList = oTest.DesignStepFactory.NewList("");
var oStepFactory = (QcClient.StepFactory)oRun.StepFactory;
foreach (QcClient.DesignStep oDesignStep in tsDesignStepList)
{
var oStep = (QcClient.Step)oStepFactory.AddItem(oDesignStep.StepName);
oStep.Status = "Passed";
oStep.Post();
}

Retrieve multiple accounts from Microsoft CRM 2011 Online

With Microsoft CRM 2011 online and using webservices, I am using below method in my Main.java using the OrganizationServiceStub class created by webservices call. The output retrieved no of records is -1 can someone help where I am going wrong. I want to retrieve the accounts where name begins with "Tel" without giving the accountid. I can see the data exists in CRM.
Thanks
public static void getAccountDetails(OrganizationServiceStub service, ArrayOfstring fields)
{
try{
ArrayOfanyType aa = new ArrayOfanyType();
aa.setAnyType(new String[] {"Tel"});
ConditionExpression condition1 = new ConditionExpression();
condition1.setAttributeName("name");
condition1.setOperator(ConditionOperator.BeginsWith);
condition1.setValues(aa);
ArrayOfConditionExpression ss = new ArrayOfConditionExpression();
ss.setConditionExpression(new ConditionExpression[] {condition1});
FilterExpression filter1 = new FilterExpression();
filter1.setConditions(ss);
QueryExpression query = new QueryExpression();
query.setEntityName("account");
ColumnSet cols = new ColumnSet();
cols.setColumns(fields);
query.setColumnSet(cols);
query.setCriteria(filter1);
RetrieveMultiple ll = new RetrieveMultiple();
ll.setQuery(query);
RetrieveMultipleResponse result1 = service.retrieveMultiple(ll);
EntityCollection accounts = result1.getRetrieveMultipleResult();
System.out.println(accounts.getTotalRecordCount());
}
catch (IOrganizationService_RetrieveMultiple_OrganizationServiceFaultFault_FaultMessage e) {
logger.error(e.getMessage());
e.printStackTrace();
}
catch (RemoteException e) {
logger.error(e.getMessage());
e.printStackTrace();
}
}

For Java, include this code snippet works for the above issue
ArrayOfanyType aa = new ArrayOfanyType();
aa.setAnyType(new String[] {"555"});
ConditionExpression condition1 = new ConditionExpression();
condition1.setAttributeName("telephone1");
condition1.setOperator(ConditionOperator.BeginsWith);
condition1.setValues(aa);
ArrayOfConditionExpression ss = new ArrayOfConditionExpression();
ss.setConditionExpression(new ConditionExpression[] {condition1});
FilterExpression filter1 = new FilterExpression();
filter1.setConditions(ss);
QueryExpression query = new QueryExpression();
query.setEntityName("account");
PagingInfo pagingInfo = new PagingInfo();
pagingInfo.setReturnTotalRecordCount(true);
query.setPageInfo(pagingInfo);
OrganizationServiceStub.ColumnSet colSet = new OrganizationServiceStub.ColumnSet();
OrganizationServiceStub.ArrayOfstring cols = new OrganizationServiceStub.ArrayOfstring();
cols.setString(new String[]{"name", "telephone1", "address1_city"});
colSet.setColumns(cols);
query.setColumnSet(colSet);
query.setCriteria(filter1);
RetrieveMultiple ll = new RetrieveMultiple();
ll.setQuery(query);
OrganizationServiceStub.RetrieveMultipleResponse response = serviceStub.retrieveMultiple(ll);
EntityCollection result = response.getRetrieveMultipleResult();
ArrayOfEntity attributes = result.getEntities();
Entity[] keyValuePairs = attributes.getEntity();
for (int i = 0; i < keyValuePairs.length; i++) {
OrganizationServiceStub.KeyValuePairOfstringanyType[] keyValuePairss = keyValuePairs[i].getAttributes().getKeyValuePairOfstringanyType();
for (int j = 0; j < keyValuePairss.length; j++) {
System.out.print(keyValuePairss[j].getKey() + ": ");
System.out.println(keyValuePairss[j].getValue());
}
}

Not sure how similar your EntityCollection object is to the .Net version in the SDK, however you need to specify ReturnTotalRecordCount in the query's PagingInfo in .Net for the TotalRecordCount property to have a value. Could you not instead check accounts.Entities.Count?
Note: I'm not a Java guy either...