Develop Reference

Web scraping Google Images with Java

I have been tasked with making an application that the user uses to search for three pictures of theirQuery . It could be written in any language, and Python was suggested. However, my GUI skills with Python suck, I am most familiar with Java, and I have downloaded the Eclipse SWT package, which I intend on using.
OK, what have you done so far
I pimped their example code that makes a simple web browser. I have it display Google Images as the home page, and intend on using JavaScript/jQuery to automate the filling in of theirQuery, and returning the first three pictures. (I also plan on somehow using JavaScript/jQuery to return the three pictures.)
According to the documentation for Eclipse SWT, evaluate(),execute() are used to evaluate,execute JavaScript: http://help.eclipse.org/indigo/index.jsp?topic=/org.eclipse.platform.doc.isv/reference/api/org/eclipse/swt/browser/Browser.html
Here is the code that I have used:
import org.eclipse.swt.*;
import org.eclipse.swt.browser.Browser;
import org.eclipse.swt.browser.LocationEvent;
import org.eclipse.swt.browser.LocationListener;
import org.eclipse.swt.browser.ProgressEvent;
import org.eclipse.swt.browser.ProgressListener;
import org.eclipse.swt.browser.StatusTextEvent;
import org.eclipse.swt.layout.GridData;
import org.eclipse.swt.layout.GridLayout;
import org.eclipse.swt.widgets.Display;
import org.eclipse.swt.widgets.Event;
import org.eclipse.swt.widgets.Label;
import org.eclipse.swt.widgets.Listener;
import org.eclipse.swt.widgets.ProgressBar;
import org.eclipse.swt.widgets.Shell;
import org.eclipse.swt.widgets.Text;
import org.eclipse.swt.widgets.ToolBar;
import org.eclipse.swt.widgets.ToolItem;
public class BrowserCodeDemo {
public static void main(String[] args) {
Display display = new Display();
final Shell shell = new Shell(display);
GridLayout gridLayout = new GridLayout();
gridLayout.numColumns = 3;
shell.setLayout(gridLayout);
ToolBar toolbar = new ToolBar(shell, SWT.NONE);
ToolItem itemBack = new ToolItem(toolbar, SWT.PUSH);
itemBack.setText("Back");
ToolItem itemForward = new ToolItem(toolbar, SWT.PUSH);
itemForward.setText("Forward");
ToolItem itemStop = new ToolItem(toolbar, SWT.PUSH);
itemStop.setText("Stop");
ToolItem itemRefresh = new ToolItem(toolbar, SWT.PUSH);
itemRefresh.setText("Refresh");
ToolItem itemGo = new ToolItem(toolbar, SWT.PUSH);
itemGo.setText("Go");
GridData data = new GridData();
data.horizontalSpan = 3;
toolbar.setLayoutData(data);
Label labelAddress = new Label(shell, SWT.NONE);
labelAddress.setText("Address");
final Text location = new Text(shell, SWT.BORDER);
data = new GridData();
data.horizontalAlignment = GridData.FILL;
data.horizontalSpan = 2;
data.grabExcessHorizontalSpace = true;
location.setLayoutData(data);
final Browser browser;
try {
browser = new Browser(shell, SWT.NONE);
} catch (SWTError e) {
System.out.println("Could not instantiate Browser: " + e.getMessage());
display.dispose();
return;
}
data = new GridData();
data.horizontalAlignment = GridData.FILL;
data.verticalAlignment = GridData.FILL;
data.horizontalSpan = 3;
data.grabExcessHorizontalSpace = true;
data.grabExcessVerticalSpace = true;
browser.setLayoutData(data);
final Label status = new Label(shell, SWT.NONE);
data = new GridData(GridData.FILL_HORIZONTAL);
data.horizontalSpan = 2;
status.setLayoutData(data);
final ProgressBar progressBar = new ProgressBar(shell, SWT.NONE);
data = new GridData();
data.horizontalAlignment = GridData.END;
progressBar.setLayoutData(data);
/* event handling */
Listener listener = new Listener() {
#Override
public void handleEvent(Event event) {
ToolItem item = (ToolItem)event.widget;
String string = item.getText();
if (string.equals("Back")) browser.back();
else if (string.equals("Forward")) browser.forward();
else if (string.equals("Stop")) browser.stop();
else if (string.equals("Refresh")) browser.refresh();
else if (string.equals("Go")) browser.setUrl(location.getText());
}
};
browser.addProgressListener(new ProgressListener() {
#Override
public void changed(ProgressEvent event) {
if (event.total == 0) return;
int ratio = event.current * 100 / event.total;
progressBar.setSelection(ratio);
}
#Override
public void completed(ProgressEvent event) {
progressBar.setSelection(0);
}
});
browser.addLocationListener(new LocationListener() {
#Override
public void changed(LocationEvent event) {
if (event.top) location.setText(event.location);
}
#Override
public void changing(LocationEvent event)
{
//System.out.println("Something is happening.");
}
});
itemBack.addListener(SWT.Selection, listener);
itemForward.addListener(SWT.Selection, listener);
itemStop.addListener(SWT.Selection, listener);
itemRefresh.addListener(SWT.Selection, listener);
itemGo.addListener(SWT.Selection, listener);
location.addListener(SWT.DefaultSelection, new Listener() {
#Override
public void handleEvent(Event e)
{
browser.setUrl(location.getText());
System.out.println("New URL loaded");
}
});
shell.open();
browser.setUrl(
"https://www.google.com/imghp?hl=en&tab=wi&ei=m8g4VLndMaz4igKlvoDADg&ved=0CAMQqi4oAQ");
//browser.setVisible(false);
boolean jQueryExecuted = browser.execute("$(\'#lst-ib\').val(\'snopes\')");
if (!jQueryExecuted)
{
System.out.println("Your jQuery didn't execute.");
}
jQueryExecuted = browser.execute("$(\'[name=btnG]\').click()");
if (!jQueryExecuted)
{
System.out.println("Your jQuery didn't execute.");
}
while (!shell.isDisposed()) {
if (!display.readAndDispatch())
display.sleep();
}
display.dispose();
}
}
I don't think the issue is the code itself, as I have tried to set the text field automatically and it failed, both in Java and in my browser; /* the name of the text field is "lst-ib" */
What are you trying to web-scrape?
I am trying to web scrape Google images: https://www.google.com/imghp?hl=en&tab=wi&ei=m8g4VLndMaz4igKlvoDADg&ved=0CAMQqi4oAQ .
The text field is an input with name "lst-ib". The button is an input with name "btnG". The text field is within a div with id of "gs_lc0". (I see multiple other input elements in there, too, of the exact same size).

Oh, wow. It has been 10 hours and nobody has so much as commented on here but me. It turns out that the problem is much bigger than I thought: Browser from Eclipse SWT cannot execute jQuery.
Luckily, I found the Google Image API, which is written in JavaScript, and is deprecated, but still works. Also, I found some simple source code on the page: https://developers.google.com/image-search/v1/devguide#hiworld
I changed the newImg.src = "/image-search/v1/result.tbUrl;" to newImg.src = result.tbUrl; so that the images would load. This changes everything.
What is your new plan?
I now plan on using this code from Java. I plan on changing the src to the src that is the Google Image API that the JavaScript code uses, and then invoking the code to get the first three images. This is how, apparently, you run JavaScript from Java: How to run javascript in java programming . If only I can figure out how to change the src of the JavaScript engine that Java uses to this: https://www.google.com/jsapi

Hide ON_TOP shell in SWT on Display minimimized

I'm trying to hide a SWT shell when the Display is minimized. I'm missing something and would be most thankful for any help.
Additional Info: This shell is actually a popup that gets drawn when the user clicks on a composite. In the end, my goal is to hide this popup-shell when the composite is not visible (user minimized the window or switched between windows, say with Alt+Tab for example).
Here's my code:
static Shell middleClickNodeInfoShell ;
static Label nodeIdLabel ;
void init(){
...
/** Focused node on middle click*/
middleClickNodeInfoShell = new Shell(Display.getDefault(), SWT.BORDER | SWT.MODELESS);
middleClickNodeInfoShell.setLayoutData(new GridData(GridData.FILL, GridData.BEGINNING, true, false));
middleClickNodeInfoShell.setLayout(createNoMarginLayout(1, false));
nodeIdLabel = new Label(middleClickNodeInfoShell, SWT.NONE);
Display.getDefault().addListener(SWT.Iconify,new Listener() {
#Override
public void handleEvent(Event arg0) {
// TODO Auto-generated method stub
middleClickNodeInfoShell.setVisible(false);
}
});
}
#Override
public boolean onMouseClicked(Button button, ScreenPosition screenPos,
final GeoPosition arg2) {
...
nodeIdLabel.setText("Node Id: "+node.getId());
middleClickNodeInfoShell.setLocation(pos.getX()+displayX,pos.getY()+displayY+30);
middleClickNodeInfoShell.setVisible(true);
middleClickNodeInfoShell.pack();
}

Here is sample code that will help you do figure out what you are looking for
public static void main(String[] args) {
final Display display = new Display();
final Shell shell = new Shell(display);
shell.setSize(300, 200);
shell.setText("Shell Example");
shell.setLayout(new RowLayout());
final Button button = new Button(shell, SWT.PUSH);
button.setText("Click Me");
final Shell tip = new Shell(shell,SWT.MODELESS);
tip.setLayout(new FillLayout());
Label lbl = new Label(tip, SWT.NONE);
lbl.setText("***tooltip***");
tip.pack();
shell.addControlListener(new ControlListener() {
#Override
public void controlResized(ControlEvent e) {
changeTipLocation(display, button, tip);
}
#Override
public void controlMoved(ControlEvent e) {
changeTipLocation(display, button, tip);
}
});
button.addSelectionListener(new SelectionAdapter() {
public void widgetSelected(SelectionEvent event) {
changeTipLocation(display, button, tip);
tip.open();
}
});
shell.open();
while (!shell.isDisposed()) {
if (!display.readAndDispatch())
display.sleep();
}
display.dispose();
}
private static void changeTipLocation(final Display display, final Button button, final Shell tip) {
Rectangle bounds = button.getBounds();
Point loc = button.getLocation();
tip.setLocation(display.map(button, null, new Point(loc.x+bounds.width, loc.y+bounds.height)));
}

How to show up an image with swt in java?

My try as follows,which doesn't come up with anything:
public static void main(String[] args) {
Display display = new Display();
Shell shell = new Shell(display);
Image image = new Image(display,
"D:/topic.png");
GC gc = new GC(image);
gc.setForeground(display.getSystemColor(SWT.COLOR_WHITE));
gc.drawText("I've been drawn on",0,0,true);
gc.dispose();
shell.pack();
shell.open();
while (!shell.isDisposed()) {
if (!display.readAndDispatch()) {
display.sleep();
}
}
display.dispose();
// TODO Auto-generated method stub
}

See the SWT-Snippets for examples. This one uses an image label
Shell shell = new Shell (display);
Label label = new Label (shell, SWT.BORDER);
label.setImage (image);

You are missing one thing in your code. Event Handler for paint. Normally when you create a component it generates a paint event. All the drawing related stuff should go in it.
Also you need not to create the GC explicitly.. It comes with the event object :)
import org.eclipse.swt.*;
import org.eclipse.swt.graphics.*;
import org.eclipse.swt.layout.*;
import org.eclipse.swt.widgets.*;
public class ImageX
{
public static void main (String [] args)
{
Display display = new Display ();
Shell shell = new Shell (display, SWT.SHELL_TRIM | SWT.DOUBLE_BUFFERED);
shell.setLayout(new FillLayout ());
final Image image = new Image(display, "C:\\temp\\flyimage1.png");
shell.addListener (SWT.Paint, new Listener ()
{
public void handleEvent (Event e) {
GC gc = e.gc;
int x = 10, y = 10;
gc.drawImage (image, x, y);
gc.dispose();
}
});
shell.setSize (600, 400);
shell.open ();
while (!shell.isDisposed ()) {
if (!display.readAndDispatch ())
display.sleep ();
}
if(image != null && !image.isDisposed())
image.dispose();
display.dispose ();
}
}

Display parent modal dialog with SWT

AWT/Swing allows to show application modal (blocking the whole application) and parent modal (blocking only the parents) dialogs. How can I achieve the same with SWT?

In order to block the whole application, you can create the dialog Shell with the style SWT.APPLICATION_MODAL, open it, and then pump the UI events until the shell is disposed:
Display display = Display.getDefault();
Shell dialogShell = new Shell(display, SWT.APPLICATION_MODAL);
// populate dialogShell
dialogShell.open();
while (!dialogShell.isDisposed()) {
if (!display.readAndDispatch()) {
display.sleep();
}
}
If you want to block input only to the parent, try using the style SWT.PRIMARY_MODAL, though the Javadocs specify (as for the other modal styles) that this is a hint; i.e., that different SWT implementations may not exactly handle it the same way. Likewise, I don't know of an implementation that would honor the SWT.SYSTEM_MODAL style.
UPDATE: Answer to first comment
If you have two or more primary modals open at the same time, you cannot use the tricks to pump the events until the modal is closed, as they could be closed in any order. The code will run, but execution will resume after the while loop after the current dialog is closed and all other such dialogs that have been opened after it. In this case, I would register a DisposeListener on each dialog to get a callback when they are closed. Something like this:
void run() {
Display display = new Display();
Shell shell1 = openDocumentShell(display);
Shell shell2 = openDocumentShell(display);
// close both shells to exit
while (!shell1.isDisposed() || !shell2.isDisposed()) {
if (!display.readAndDispatch()) {
display.sleep();
}
}
display.dispose();
}
Shell openDocumentShell(final Display display) {
final Shell shell = new Shell(display, SWT.SHELL_TRIM);
shell.setLayout(new FillLayout());
Button button = new Button(shell, SWT.PUSH);
button.setText("Open Modal Dialog");
button.addSelectionListener(new SelectionAdapter() {
#Override
public void widgetSelected(SelectionEvent e) {
System.out.println("Button pressed, about to open modal dialog");
final Shell dialogShell = new Shell(shell, SWT.PRIMARY_MODAL | SWT.SHEET);
dialogShell.setLayout(new FillLayout());
Button closeButton = new Button(dialogShell, SWT.PUSH);
closeButton.setText("Close");
closeButton.addSelectionListener(new SelectionAdapter() {
#Override
public void widgetSelected(SelectionEvent e) {
dialogShell.dispose();
}
});
dialogShell.setDefaultButton(closeButton);
dialogShell.addDisposeListener(new DisposeListener() {
#Override
public void widgetDisposed(DisposeEvent e) {
System.out.println("Modal dialog closed");
}
});
dialogShell.pack();
dialogShell.open();
}
});
shell.pack();
shell.open();
return shell;
}

Windows 7 Seems to break SWT Control.print(GC)

A bug has been filed and fixed (super quickly) in SWT:
https://bugs.eclipse.org/bugs/show_bug.cgi?id=305294
Just to preface this, my goal here is to print the two images into a canvas so that I can animate the canvas sliding across the screen (think iPhone), sliding the controls themselves was too CPU intensive, so this was a good alternative until I tested it on Win7. I'm open to anything that will help me solve my original problem, it doesn't have to be fixing the problem below.
Does anyone know how to get "Control.print(GC)" to work with Windows 7 Aero? I have code that works just fine in Windows XP and in Windows 7, when Aero is disabled, but the command:
control.print(GC) causes a non-top control to be effectively erased from the screen.
GC gc = new GC(image);
try {
// As soon as this code is called, calling "layout" on the controls
// causes them to disappear.
control.print(gc);
} finally {
gc.dispose();
}
I have stacked controls and would like to print the images from the current and next controls such that I can "slide" them off the screen. However, upon printing the non-top control, it is never redrawn again.
Here is some example code. (Interesting code bits are at the top and it will require pointing at SWT in order to work.)
Thanks for any and all help. As a work around, I'm thinking about swapping controls between prints to see if that helps, but I'd rather not.
import org.eclipse.swt.SWT;
import org.eclipse.swt.custom.StackLayout;
import org.eclipse.swt.events.SelectionAdapter;
import org.eclipse.swt.events.SelectionEvent;
import org.eclipse.swt.graphics.GC;
import org.eclipse.swt.graphics.Image;
import org.eclipse.swt.graphics.Point;
import org.eclipse.swt.layout.GridData;
import org.eclipse.swt.layout.GridLayout;
import org.eclipse.swt.widgets.Button;
import org.eclipse.swt.widgets.Composite;
import org.eclipse.swt.widgets.Control;
import org.eclipse.swt.widgets.Display;
import org.eclipse.swt.widgets.Label;
import org.eclipse.swt.widgets.Shell;
public class SWTImagePrintTest {
private Composite stack;
private StackLayout layout;
private Label lblFlip;
private Label lblFlop;
private boolean flip = true;
private Button buttonFlop;
private Button buttonPrint;
/**
* Prints the control into an image
*
* #param control
*/
protected void print(Control control) {
Image image = new Image(control.getDisplay(), control.getBounds());
GC gc = new GC(image);
try {
// As soon as this code is called, calling "layout" on the controls
// causes them to disappear.
control.print(gc);
} finally {
gc.dispose();
}
}
/**
* Swaps the controls in the stack
*/
private void flipFlop() {
if (flip) {
flip = false;
layout.topControl = lblFlop;
buttonFlop.setText("flop");
stack.layout();
} else {
flip = true;
layout.topControl = lblFlip;
buttonFlop.setText("flip");
stack.layout();
}
}
private void createContents(Shell shell) {
shell.setLayout(new GridLayout(2, true));
stack = new Composite(shell, SWT.NONE);
GridData gdStack = new GridData(GridData.FILL_BOTH);
gdStack.horizontalSpan = 2;
stack.setLayoutData(gdStack);
layout = new StackLayout();
stack.setLayout(layout);
lblFlip = new Label(stack, SWT.BOLD);
lblFlip.setBackground(Display.getCurrent().getSystemColor(
SWT.COLOR_CYAN));
lblFlip.setText("FlIp");
lblFlop = new Label(stack, SWT.NONE);
lblFlop.setBackground(Display.getCurrent().getSystemColor(
SWT.COLOR_BLUE));
lblFlop.setText("fLoP");
layout.topControl = lblFlip;
stack.layout();
buttonFlop = new Button(shell, SWT.FLAT);
buttonFlop.setText("Flip");
GridData gdFlip = new GridData();
gdFlip.horizontalAlignment = SWT.RIGHT;
buttonFlop.setLayoutData(gdFlip);
buttonFlop.addSelectionListener(new SelectionAdapter() {
#Override
public void widgetSelected(SelectionEvent e) {
flipFlop();
}
});
buttonPrint = new Button(shell, SWT.FLAT);
buttonPrint.setText("Print");
GridData gdPrint = new GridData();
gdPrint.horizontalAlignment = SWT.LEFT;
buttonPrint.setLayoutData(gdPrint);
buttonPrint.addSelectionListener(new SelectionAdapter() {
#Override
public void widgetSelected(SelectionEvent e) {
print(lblFlip);
print(lblFlop);
}
});
}
/**
* #param args
*/
public static void main(String[] args) {
Shell shell = new Shell();
shell.setText("Slider Test");
shell.setSize(new Point(800, 600));
shell.setLayout(new GridLayout());
SWTImagePrintTest tt = new SWTImagePrintTest();
tt.createContents(shell);
shell.open();
Display display = Display.getDefault();
while (shell.isDisposed() == false) {
if (display.readAndDispatch() == false) {
display.sleep();
}
}
display.dispose();
}
}

This was the result of a bug:
https://bugs.eclipse.org/bugs/show_bug.cgi?id=305294
It's since been fixed. Hopefully a new packaged version of SWT will come out soon so we can officially use it.