Jsoup parsing google results - java

I want to grab all the links and thumbnails wherever given from a particular google search. Here is my code.
package com.esocial.util;
import java.io.IOException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
public class ListLinks {
public static void main(String[] args) throws IOException {
String url = "https://www.google.co.in/webhp?sourceid=chrome-instant&rlz=1C1CHWA_enIN609IN609&ion=1&espv=2&ie=UTF-8#q=thermodynamics%20cbse";
System.out.println("Fetching : "+url+"\n\n");
Document doc = Jsoup.connect(url).userAgent("Mozilla").get();
Elements div = doc.select("div.srg");
for(Element di : div)
{
Elements lists = di.select("li.g");
for(Element list : lists)
{
Element anc = list.select("a").first();
Element img = list.select("img").first();
System.out.println("\nLink : "+anc.attr("href")+"\nImage Link : "+img.attr("src")+"\n------------------------------------------\n");
}
}
}
}
But this code is not running properly and does not display results. I don't understand what the problem is.

I have also worked on to grab the search result.
To get html page . I requested a post REST call.
// HTTP GET request
private void sendGet(String query, GoogleSearchCallback googleSearchCallback) throws Exception {
ArrayList<GoogleSearchResult> googleSearchResults = new ArrayList<>();
String url = "https://www.google.co.in/search?q=" + query + "#q=" + query + "course";
URL obj = new URL(url);
HttpURLConnection con = (HttpURLConnection) obj.openConnection();
// optional default is GET
con.setRequestMethod("GET");
con.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64)");
int responseCode = con.getResponseCode();
System.out.println("\nSending 'GET' request to URL : " + url);
System.out.println("Response Code : " + responseCode);
BufferedReader in = new BufferedReader(new InputStreamReader(
con.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
// print result
System.out.println(response.toString());
Document document = Jsoup.parse(response.toString());
Elements links = document.select("a[href]");
for (Element link : links) {
String temp = link.attr("href");
if (temp.startsWith("/url?q=")) {
Log.i(TAG, "" + temp.replace("/url?q=", "") + "");
URL tempUrl = new URL(temp.replace("/url?q=", ""));
String path = tempUrl.getFile().substring(0, tempUrl.getFile().lastIndexOf('/'));
String base = tempUrl.getProtocol() + "://" + tempUrl.getHost() + path;
Log.i(TAG, "" + base);
Log.e(TAG, getDomainName(temp));
GoogleSearchResult googleSearchResult = new GoogleSearchResult();
googleSearchResult.setResultLink(base);
googleSearchResult.setResultTitle(link.text());
googleSearchResults.add(googleSearchResult);
}
}
googleSearchCallback.onGetGoogleResultSuccess(googleSearchResults);
}
private String getDomainName(String url) {
String domainName = "";
matcher = patternDomainName.matcher(url);
if (matcher.find()) {
domainName = matcher.group(0).toLowerCase().trim();
}
return domainName;
}
private static Pattern patternDomainName;
private Matcher matcher;
private static final String DOMAIN_NAME_PATTERN = "^(http(s)?|ftp|file)://[-a-zA-Z0-9+&##/%?=~_|!:,.;]*[-a-zA-Z0-9+&##/%=~_|]";
static {
patternDomainName = Pattern.compile(DOMAIN_NAME_PATTERN);
}

Related

How to run this java file?

I have been told to execute this java program which is linked to a virtual number, when you run it, the output is a number code or whatever it is that i sent the number in sms.
I am using selenium and maven and also in the eclipse program and was told to use the testng plugin. They asked me to use #test annotations in an .xml file to execute it.
Im knew to programming and to be honest i have no idea how to write the xml file in order to run this and when i ask the person who told me to do this all they do is say google it and i have tried but i havent found anything.
the code is:
package utility;
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import javax.net.ssl.HttpsURLConnection;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.Entity;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.json.JSONArray;
import org.json.JSONObject;
import org.junit.Assert;
import org.apache.commons.codec.binary.Base64;
//import sun.misc.BASE64Encoder;
public class APIcall {
// String MobileNo = ""; // virtual mobile no. (From message media)
private final String USER_AGENT = "Mozilla/5.0";
// Msg media Account:
// Password:
String name = ;
String password = ;
String authString = name + ":" + password;
//String authStringEnc = new BASE64Encoder().encode(authString.getBytes());
String authStringEnc = new Base64().encodeBase64String(authString.getBytes());
static String URL = "https://api.messagemedia.com";
String getURI = "/v1/replies";
static String postURI = "/v1/replies/confirmed";
static Client client = ClientBuilder.newClient();
public Data getRequest() {
Response response = (Response) client.target(URL +
getURI).request(MediaType.APPLICATION_JSON_TYPE)
.header("Authorization", "Basic " + authStringEnc).get();
String body = response.readEntity(String.class);
// System.out.println("status: " + response.getStatus());
// System.out.println("headers \n: " + response.getHeaders());
System.out.println("body: \n" + body);
JSONObject json = new JSONObject(body);
JSONArray reply = (JSONArray) json.get("replies");
List<String> ids = new ArrayList<String>();
for (int idx = 0; idx < reply.length(); idx++) {
// ids.add(replyIds.get(idx).toString());
ids.add(reply.getJSONObject(idx).getString("reply_id"));
}
Assert.assertEquals(200, response.getStatus());
/*
* String OTPcode = (String) body.substring(body.lastIndexOf(' ') + 1); OTPcode
* = OTPcode.substring(0, 6).trim();
*/
String getContent = reply.getJSONObject(0).getString("content");
System.out.println(getContent);
String[] content = getContent.split(" ");
String code = content[content.length - 1];
// System.out.println("OTP code : " + code);
// System.out.println("List of replies : " + ids);
return new Data(code, ids, getContent);
//JSONException
}
public final class Data {
public String otp;
public List<String> list;
public String getContent;
public Data(String otp, List list, String getContent) {
this.otp = otp;
this.list = list;
this.getContent = getContent;
}
}
public void postRequest() {
/*
* Post request : Add reply ids recieved from GET request and pass it in body
* with POST request to confirm replies
*/
List<String> ids = getRequest().list;
String postStr = "{ \"reply_ids\":[";
for (String id : ids) {
postStr += "\"" + id + "\", ";
}
postStr += " ]}";
StringBuilder postString = new StringBuilder(postStr);
postString.replace(postStr.lastIndexOf(","),
postStr.lastIndexOf(",") + 1, "");
postStr = postString.toString();
// System.out.println("Reply id list : " + postStr);
Response response = client.target(URL + postURI)
.request(MediaType.APPLICATION_JSON_TYPE)
.header("Accept", "application/json")
.header("Authorization", "Basic " +
authStringEnc).header("Content-Type",
"application/json").post(Entity.json(postStr));
/*
* System.out.println("status: " + response.getStatus());
* System.out.println("headers \n: " + response.getHeaders());
* System.out.println("body: \n" +
response.readEntity(String.class));
*/
}
private void sendGet() throws Exception {
String url = URL + getURI;
URL obj = new URL(url);
HttpURLConnection con = (HttpURLConnection) obj.openConnection();
// optional default is GET
con.setRequestMethod("GET");
// add request header
con.setRequestProperty("User-Agent", USER_AGENT);
con.setRequestProperty("Accept", "application/json");
con.setRequestProperty("Authorization", "Basic " + authStringEnc);
int responseCode = con.getResponseCode();
System.out.println("\nSending 'GET' request to URL : " + url);
System.out.println("Response Code : " + responseCode);
BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
// print result
System.out.println(response.toString());
}
private void sendPost() throws Exception {
String url = URL + postURI;
URL obj = new URL(url);
HttpsURLConnection con = (HttpsURLConnection) obj.openConnection();
// add reuqest header
con.setRequestMethod("POST");
con.setRequestProperty("User-Agent", USER_AGENT);
// con.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
con.setRequestProperty("Accept", "application/json");
con.setRequestProperty("Authorization", "Basic " + authStringEnc);
String urlParameters = "{\"reply_ids\":\"25ebbf01-5614-4ea6-a4f7-67b752d18ed2\",\"63ed8e18-ecf0-444d-b79e-341e944b0b94\"}";
// Send post request
con.setDoOutput(true);
DataOutputStream wr = new DataOutputStream(con.getOutputStream());
wr.writeBytes(urlParameters);
wr.flush();
wr.close();
int responseCode = con.getResponseCode();
System.out.println("\nSending 'POST' request to URL : " + url);
System.out.println("Post parameters : " + urlParameters);
System.out.println("Response Code : " + responseCode);
BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
// print result
System.out.println(response.toString());
}
public static void main(String[] args) throws Exception {
/*
* System.setProperty("http.proxyHost", "");
* System.setProperty("http.proxyPort", "");
*/
APIcall apiCall = new APIcall();
apiCall.getRequest();
apiCall.postRequest();
}
}
If I understand correctly, they want you to convert this into a TestNG test which shouldn't be too difficult as the test and methods are already built. Considering you don't know Java, nor programming, I suggest watching a couple of Youtube videos to get you up to speed on TestNG. I believe once you get your feet wet with TestNG, much of the code along with how to convert it will make more sense.
#Test annotation will behave like a void main for each test (which is a method with the #Test annotation).
You can add a dependency in your maven pom.xml file for junit or testng (any of them will bring this annotation), then, you can trigger the test by running the method from your IDE, or from command line you can simple use:
mvn clean test - to trigger all existent tests
mvn clean test -Dtest=your.package.TestClassName - to trigger tests from a class
mvn clean test -Dtest=your.package.TestClassName#particularMethod - to trigger a specific test

how's model of post request to login gmail in java

i tried to login gmail via sending post request flowwing the code:
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package testautologingmail;
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.CookieHandler;
import java.net.CookieManager;
import java.net.HttpCookie;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import javax.net.ssl.HttpsURLConnection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
public class HttpUrlConnectionExample {
private static List<String> cookies = new ArrayList<>();;
private static HttpsURLConnection conn;
private final String USER_AGENT = "Mozilla/5.0";
public HttpUrlConnectionExample(){
}
public static void main(String[] args) throws Exception {
String url = "https://accounts.google.com/ServiceLoginAuth";
String gmail = "https://mail.google.com/mail/";
HttpUrlConnectionExample http = new HttpUrlConnectionExample();
// make sure cookies is turn on
//CookieHandler.setDefault(new CookieManager());
// 1. Send a "GET" request, so that you can extract the form's data.
String page = http.GetPageContent(url);
String postParams = http.getFormParams(page,"kiemtienol1506","vandai1507");
// 2. Construct above post's content and then send a POST request for
// authentication
http.sendPost(url, postParams);
// 3. success then go to gmail.
String result = http.GetPageContent(gmail);
System.out.println(result);
conn.disconnect();
}
private static void getcookies(String url) throws MalformedURLException, IOException{
CookieManager cookieManager = new CookieManager();
CookieHandler.setDefault(cookieManager);
URL obj = new URL(url);
conn = (HttpsURLConnection) obj.openConnection();
conn.getContent();
List<HttpCookie> cookies2 = cookieManager.getCookieStore().getCookies();
System.out.println("get cookies");
for (HttpCookie cookie : cookies2) {
System.out.println(cookie.getDomain());
System.out.println(cookie.toString());
String temp = cookie.toString();
cookies.add(temp);
}
//conn.disconnect();
}
private void sendPost(String url, String postParams) throws Exception {
URL obj = new URL(url);
conn = (HttpsURLConnection) obj.openConnection();
// Acts like a browser
conn.setUseCaches(false);
conn.setRequestMethod("POST");
conn.setRequestProperty("Host", "accounts.google.com");
conn.setRequestProperty("User-Agent", USER_AGENT);
conn.setRequestProperty("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
System.out.println("cookies String");
for (String cookie : cookies) {
//String[] tmp = cookie.split(":");
conn.addRequestProperty("Cookie", cookie.split(";", 1)[0]);
System.out.println(cookie.split(";", 1)[0]);
}
conn.setRequestProperty("Connection", "keep-alive");
conn.setRequestProperty("Referer", "https://accounts.google.com/ServiceLoginAuth");
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
conn.setRequestProperty("Content-Length", Integer.toString(postParams.length()));
conn.setDoOutput(true);
conn.setDoInput(true);
// Send post request
DataOutputStream wr = new DataOutputStream(conn.getOutputStream());
wr.writeBytes(postParams);
wr.flush();
wr.close();
int responseCode = conn.getResponseCode();
System.out.println("\nSending 'POST' request to URL : " + url);
System.out.println("Post parameters : " + postParams);
System.out.println("Response Code : " + responseCode);
BufferedReader in =
new BufferedReader(new InputStreamReader(conn.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
//System.out.println(response.toString());
//conn.disconnect();
}
private String GetPageContent(String url) throws Exception {
URL obj = new URL(url);
conn = (HttpsURLConnection) obj.openConnection();
// default is GET
conn.setRequestMethod("GET");
conn.setUseCaches(false);
// act like a browser
conn.setRequestProperty("User-Agent", USER_AGENT);
conn.setRequestProperty("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
if (cookies != null) {
for (String cookie : cookies) {
conn.addRequestProperty("Cookie", cookie.split(";", 1)[0]);
}
}
int responseCode = conn.getResponseCode();
System.out.println("\nSending 'GET' request to URL : " + url);
System.out.println("Response Code : " + responseCode);
BufferedReader in =
new BufferedReader(new InputStreamReader(conn.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
// Get the response cookies
//setCookies(conn.getHeaderFields().get("Set-Cookie"));
getcookies(url);
return response.toString();
}
public String getFormParams(String html, String username, String password)
throws UnsupportedEncodingException {
System.out.println("Extracting form's data...");
Document doc = Jsoup.parse(html);
// Google form id
Element loginform = doc.getElementById("gaia_loginform");
Elements inputElements = loginform.getElementsByTag("input");
List<String> paramList = new ArrayList<String>();
for (Element inputElement : inputElements) {
String key = inputElement.attr("name");
String value = inputElement.attr("value");
String id = inputElement.attr("id");
if (key.equals("Email"))
{
value = username;
paramList.add(key + "=" + URLEncoder.encode(value, "UTF-8"));
}
else if (key.equals("Passwd")){
paramList.add(key + "=" + URLEncoder.encode(value, "UTF-8"));
value = password;
}
else if(id.equals("Passwd-hidden")){
value = password;
paramList.add("Passwd=" + URLEncoder.encode(value, "UTF-8"));
}else
paramList.add(key + "=" + URLEncoder.encode(value, "UTF-8"));
}
// build parameters list
StringBuilder result = new StringBuilder();
for (String param : paramList) {
if (result.length() == 0) {
result.append(param);
} else {
result.append("&" + param);
}
}
return result.toString();
}
public List<String> getCookies() {
return cookies;
}
public void setCookies(List<String> cookies) {
this.cookies = cookies;
}
}
i also try to get cookies flow this line setCookies(conn.getHeaderFields().get("Set-Cookie")); but it's always return null String so i have to make getCookies() function.
i always receive status 200 OK. So is there any support to help me or another code can do this task.
Thanks in Advance.

How to inclosed a code in a function

This is my java code i am repeating the same steps for url and url1 so i want make a function in which i place the my url code seperate and url1 code seperate then call it in a main class. First I want to access String url and then I want to access String url1.As I am new in java so I am new in java so I am not able to enclose it into function
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.json.JSONArray;
import org.json.JSONObject;
public class Test_URL_Req {
public static void main(String[] args){
// TODO Auto-generated method stub
try {
String id ="301";
String url = "https://tfs.tpsonline.com/IRIS%204.0%20Collection/Main/_apis/build/definitions?api-version=4.1";
String url1 ="https://tfs.tpsonline.com/IRIS%204.0%20Collection/Main/_apis/build/builds?api-version=4.1&definitions=" + id +"&resultFilter=succeeded&$top=1";
URL obj = new URL(url);
URL obj1 = new URL(url1);
HttpURLConnection con = (HttpURLConnection) obj.openConnection();
HttpURLConnection con1 = (HttpURLConnection) obj1.openConnection();
int responseCode = con.getResponseCode();
int responseCode1 = con1.getResponseCode();
System.out.println("\nSending 'GET' request to URL : " + url);
System.out.println("Response Code : " + responseCode);
System.out.println("\nSending 'GET' request to URL : " + url1);
System.out.println("Response Code : " + responseCode1);
BufferedReader in = new BufferedReader(
new InputStreamReader(con.getInputStream()));
BufferedReader in1 = new BufferedReader(
new InputStreamReader(con1.getInputStream()));
String inputLine;
String inputLine1;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
//System.out.println(response);
}
StringBuffer response1 = new StringBuffer();
while ((inputLine1 = in1.readLine()) != null) {
response1.append(inputLine1);
//System.out.println(response1);
}
in.close();
JSONObject obj_JSONObject = new JSONObject(response.toString());
JSONObject obj_JSONObject1 = new JSONObject(response1.toString());
JSONArray obj_JSONArray = obj_JSONObject.getJSONArray("value");
JSONArray obj_JSONArray1 = obj_JSONObject1.getJSONArray("value");
for(int i=0; i<obj_JSONArray.length();i++)
{
JSONObject obj_JSONObject2 = obj_JSONArray.getJSONObject(i);
String value = obj_JSONObject2.getString("name");
//String value = obj_JSONObject2.get("id").toString();
//System.out.println(value);
String toSearch= "DEVOPS";
if(value.equals(toSearch)){
System.out.println("STATUS:-");
System.out.println(value);
String result =obj_JSONObject2.getString("name");
System.out.println("BUILD NAME");
System.out.println(result);
String Def_id = obj_JSONObject2.get("id").toString();
System.out.println("DEFINATION ID");
System.out.println(Def_id);
break;
}
}
for(int i=0; i<obj_JSONArray1.length();i++)
{
JSONObject obj_JSONObject2 = obj_JSONArray1.getJSONObject(i);
String value = obj_JSONObject2.getString("result");
//String value = obj_JSONObject2.get("id").toString();
//System.out.println(value);
String toSearch1= "succeeded";
if(value.equals(toSearch1)){
System.out.println("#######################################");
System.out.println("RESULT");
System.out.println(value);
String result =obj_JSONObject2.getString("status");
System.out.println("STATUS");
System.out.println(result);
String Def_id = obj_JSONObject2.get("id").toString();
System.out.println("BUILD ID");
System.out.println(Def_id);
boolean keepForever =obj_JSONObject2.getBoolean("keepForever");
if(keepForever == false)
{
keepForever=true;
}
System.out.println(keepForever);
}
}
} catch (Exception e) {
System.out.println(e);
}
}
}
public static String getURLResponse( String url){
try {
System.out.println("\nSending 'GET' request to URL : " + url);
URL obj = new URL(url);
HttpURLConnection con = (HttpURLConnection) obj.openConnection();
int responseCode = con.getResponseCode();
System.out.println("Response Code : " + responseCode);
BufferedReader in = new BufferedReader(
new InputStreamReader(con.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
//System.out.println(response);
}
in.close();
return response.toString();
} catch (Exception e) {
System.out.println(e);
}
return null;
}
In main method -
public static void main(String[] args){
String url = "....";
String url1 =".....";
String response = getURLResponse(url);
String response1 = getURLResponse(url1);
JSONObject obj_JSONObject = new JSONObject (response);
JSONObject obj_JSONObject1 = new JSONObject(response1);
...
}
Create a method that takes a String and it appears you want a StringBuffer response...
public StringBuffer doSomething(String url){
URL obj = new URL(url);
HttpURLConnection con = (HttpURLConnection) obj.openConnection();
int responseCode = con.getResponseCode();
//etc
return response;
}
And just pass your two URLs to it from the main:
String url = "https://tfs.tpsonline...
StringBuffer response = doSomething(url);

How do I make a second POST to a webserver?

I have a website which I'm trying to log-on to and fill in forms automatically and then finally sending the data. The first step is achieved with the code below i.e. I can log in and then redirect to the page I'd like to fill with information.
The code gets the form name and value and changes these using GET and POST and I'd like to repeat this process but this time with multiple forms (instead of just username and password). I manage to get the form name and values and change the HTML but the issues occurs when I try to POST the new values back to the server. I don't get any error messages and the code returns the new page but the form data remains unchanged.
I can't seem to find the issue in my code, the second process works just like the first process, the only difference is, I now try to POSTdata to multiple forms.
The problem, I assume, lies either in my fillFormParams or sendPost methods. I appreciate any help!
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.CookieHandler;
import java.net.CookieManager;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import javax.net.ssl.HttpsURLConnection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.net.HttpURLConnection;
public class HttpUrlConnectionExample {
private List<String> cookies;
private List<String> cookiejar;
private static HttpURLConnection conn;
private final String USER_AGENT = "Mozilla/5.0";
public static void main(String[] args) throws Exception {
String url = "http://www3.bagerisystem.se/webbordersodervidingebagarn/Logon.aspx";
String menuPage = "http://www3.bagerisystem.se/webbordersodervidingebagarn/Meny.aspx";
HttpUrlConnectionExample http = new HttpUrlConnectionExample();
// make sure cookies is turn on
http.createCookieManager();
// 1. Send a "GET" request, so that you can extract the form's data.
String page = http.GetPageContent(url);
String postParams = http.getFormParams(page, "****", "****");
// 2. Construct above post's content and then send a POST request for
// authentication
URL obj = new URL(url);
conn = (HttpURLConnection) obj.openConnection();
http.sendPost(url, postParams,conn);
// 3. success then go to menuPage.
String result = http.GetPageContent(menuPage);
//System.out.println(result);
// 4.Send new GET request
String page2 = http.GetPageContent("http://www3.bagerisystem.se/webbordersodervidingebagarn/Retur.aspx");
//System.out.println(page2);
int[][] returVal = { { 0, 1, 2, 3, 4, 5, 6, },{ 0, 1, 2, 3, 4, 5, 6, },{ 0, 1, 2, 3, 4, 5, 6, },{ 0, 1, 2, 3, 4, 5, 6, },{ 0, 1, 2, 3, 4, 5, 6, },{ 0, 1, 2, 3, 4, 5, 6, }};
String postParams2 = http.fillFormParams(page2, returVal);
// 5. Construct above post's content and then send a POST request for
// authentication
String url1 = "http://www3.bagerisystem.se/webbordersodervidingebagarn/Retur.aspx";
URL obj1 = new URL(url1);
conn = (HttpURLConnection) obj1.openConnection();
http.sendPost(url1, postParams2,conn);
// 6. success then go to menuPage.
String resultz = http.GetPageContent("http://www3.bagerisystem.se/webbordersodervidingebagarn/Retur.aspx");
System.out.println(resultz);
}
private void sendPost(String url, String postParams,HttpURLConnection conn) throws Exception {
// Acts like a browser
conn.setUseCaches(false);
conn.setRequestMethod("POST");
conn.setRequestProperty("Host", "www3.bagerisystem.se");
conn.setRequestProperty("User-Agent", USER_AGENT);
conn.setRequestProperty("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
for (String cookie : this.cookies) {
conn.addRequestProperty("Cookie", cookie.split(";", 1)[0]);
}
conn.setRequestProperty("Connection", "keep-alive");
conn.setRequestProperty("Referer", "http://www3.bagerisystem.se/webbordersodervidingebagarn/Logon.aspx");
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
conn.setRequestProperty("Content-Length", Integer.toString(postParams.length()));
conn.setDoOutput(true);
conn.setDoInput(true);
// Send post request
DataOutputStream wr = new DataOutputStream(conn.getOutputStream());
wr.writeBytes(postParams);
wr.flush();
wr.close();
int responseCode = conn.getResponseCode();
System.out.println("\nSending 'POST' request to URL : " + url);
System.out.println("Post parameters : " + postParams);
System.out.println("Response Code : " + responseCode);
BufferedReader in =
new BufferedReader(new InputStreamReader(conn.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
// System.out.println(response.toString());
}
private String GetPageContent(String url) throws Exception {
URL obj = new URL(url);
conn = (HttpURLConnection) obj.openConnection();
// default is GET
conn.setRequestMethod("GET");
conn.setUseCaches(false);
// act like a browser
conn.setRequestPropert`enter code here`y("User-Agent", USER_AGENT);
conn.setRequestProperty("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
if (cookies != null) {
for (String cookie : this.cookies) {
conn.addRequestProperty("Cookie", cookie.split(";", 1)[0]);
}
}
int responseCode = conn.getResponseCode();
// System.out.println("\nSending 'GET' request to URL : " + url);
// System.out.println("Response Code : " + responseCode);
BufferedReader in =
new BufferedReader(new InputStreamReader(conn.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
// Get the response cookies
setCookies(conn.getHeaderFields().get("Set-Cookie"));
return response.toString();
}
public String getFormParams(String html, String username, String password)
throws UnsupportedEncodingException {
//System.out.println("Extracting form's data...");
Document doc = Jsoup.parse(html);
// form id
Element loginform = doc.getElementById("Form1");
Elements inputElements = loginform.getElementsByTag("input");
List<String> paramList = new ArrayList<String>();
for (Element inputElement : inputElements) {
String key = inputElement.attr("name");
String value = inputElement.attr("value");
if (key.equals("UserNameTextBox"))//If corresponding input form is found, enter defined username
value = username;
else if (key.equals("PasswordTextBox"))//If corresponding input form is found, enter defined password
value = password;
paramList.add(key + "=" + URLEncoder.encode(value, "UTF-8"));
}
// build parameters list
StringBuilder result = new StringBuilder();
for (String param : paramList) {
if (result.length() == 0) {
result.append(param);
} else {
result.append("&" + param);
}
}
return result.toString();
}
public List<String> getCookies() {
return cookies;
}
public void setCookies(List<String> cookies) {
this.cookies = cookies;
}
public void createCookieManager(){
if (cookies == null) {
CookieHandler.setDefault(new CookieManager());
}
}
public String fillFormParams(String html, int[][] returVal) throws UnsupportedEncodingException{
Document doc = Jsoup.parse(html);
//Form id
int row = 0;
int column = 0;
StringBuilder result = new StringBuilder();
Element inputForm = doc.getElementById("Form1");
Elements inputElements = inputForm.getElementsByTag("input");
List<String> paramList = new ArrayList<String>();
while(row <5 ){
for (Element inputElement: inputElements){
String key = inputElement.attr("name");
String value = inputElement.attr("value");
String truId = "ReturDataGrid:_ctl" +(row+2);
// "ReturDataGrid%3A_ctl2%3AtxtAntal_1" + "=" + URLEncoder.encode("1", "UTF-8")
String newId = truId + ":txtAntal_" + Integer.toString(column);
if (column == 6) {
row++;
column = (column+1)%7;
}
if(key.equals(newId)){
key = "ReturDataGrid%3A_ctl"+(row+2) +"%3AtxtAntal_1" +column;
value = Integer.toString(returVal[row][column]);
paramList.add(key + "=" + URLEncoder.encode(value, "UTF-8"));
}
}
column++;
}
for (String param : paramList) {
if (result.length() == 0) {
result.append(param);
} else {
result.append("&" + param);
}
}
return result.toString();
}
}

How to download and upload file from SharePoint Server in Java?

I am trying to write a simple example code with the help of this java-sharepoint-library
library but still i am not able to design simple program.
For development used information - https://paulryan.com.au/2014/spo-remote-authentication-rest/
Everything is well described there.
import java.io.*;
import java.net.*;
import javax.xml.parsers.*;
import javax.xml.xpath.*;
import org.w3c.dom.Document;
import org.xml.sax.*;
public class LoginManagerSharePoint {
private final String sts = "https://login.microsoftonline.com/extSTS.srf";
private final String loginContextPath = "/_forms/default.aspx?wa=wsignin1.0";
//private final String contextInfoPath = "/_api/contextinfo";
private final String sharepointContext = "xxxxxxx";
private final String reqXML = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" +
"<s:Envelope xmlns:s=\"http://www.w3.org/2003/05/soap-envelope\" " +
"xmlns:a=\"http://www.w3.org/2005/08/addressing\" " +
"xmlns:u=\"http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd\">" +
"<s:Header><a:Action s:mustUnderstand=\"1\">http://schemas.xmlsoap.org/ws/2005/02/trust/RST/Issue" +
"</a:Action><a:ReplyTo><a:Address>http://www.w3.org/2005/08/addressing/anonymous</a:Address>" +
"</a:ReplyTo><a:To s:mustUnderstand=\"1\">https://login.microsoftonline.com/extSTS.srf</a:To>" +
"<o:Security s:mustUnderstand=\"1\" " +
"xmlns:o=\"http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd\">" +
"<o:UsernameToken><o:Username>[username]</o:Username><o:Password>[password]</o:Password>" +
"</o:UsernameToken></o:Security></s:Header><s:Body><t:RequestSecurityToken " +
"xmlns:t=\"http://schemas.xmlsoap.org/ws/2005/02/trust\">" +
"<wsp:AppliesTo xmlns:wsp=\"http://schemas.xmlsoap.org/ws/2004/09/policy\">" +
"<a:EndpointReference><a:Address>[endpoint]</a:Address></a:EndpointReference>" +
"</wsp:AppliesTo><t:KeyType>http://schemas.xmlsoap.org/ws/2005/05/identity/NoProofKey" +
"</t:KeyType><t:RequestType>http://schemas.xmlsoap.org/ws/2005/02/trust/Issue" +
"</t:RequestType><t:TokenType>urn:oasis:names:tc:SAML:1.0:assertion</t:TokenType>" +
"</t:RequestSecurityToken></s:Body></s:Envelope>";
private String generateSAML() {
String saml = reqXML
.replace("[username]", "username");
saml = saml.replace("[password]", "password");
saml = saml.replace("[endpoint]", String.format("https://%s.sharepoint.com/_forms/default.aspx?wa=wsignin1.0", sharepointContext));
return saml;
}
public String getCookie() {
String token;
try {
token = requestToken();
String cookie = submitToken(token);
//System.out.println(cookie);
return cookie;
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return null;
}
private String requestToken() throws XPathExpressionException, SAXException,
ParserConfigurationException, IOException {
String saml = generateSAML();
URL u = new URL(sts);
URLConnection uc = u.openConnection();
HttpURLConnection connection = (HttpURLConnection) uc;
connection.setDoOutput(true);
connection.setDoInput(true);
connection.setRequestMethod("POST");
// http://stackoverflow.com/questions/12294274/mobile-app-for-sharepoint/12295224#12295224
connection.addRequestProperty("Content-Type",
"text/xml; charset=utf-8");
OutputStream out = connection.getOutputStream();
Writer wout = new OutputStreamWriter(out);
wout.write(saml);
wout.flush();
wout.close();
InputStream in = connection.getInputStream();
int c;
StringBuilder sb = new StringBuilder("");
while ((c = in.read()) != -1)
sb.append((char) (c));
in.close();
String result = sb.toString();
String token = extractToken(result);
//System.out.println(token);
return token;
}
private String extractToken(String result) throws SAXException, IOException, ParserConfigurationException, XPathExpressionException {
//http://stackoverflow.com/questions/773012/getting-xml-node-text-value-with-java-dom
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
DocumentBuilder db = dbf.newDocumentBuilder();
Document document = db.parse(new InputSource(new StringReader(result)));
XPathFactory xpf = XPathFactory.newInstance();
XPath xp = xpf.newXPath();
String token = xp.evaluate("//BinarySecurityToken/text()", document.getDocumentElement());
//handle error S:Fault:
//http://social.microsoft.com/Forums/en-US/crmdevelopment/thread/df862099-d9a1-40a4-b92e-a107af5d4ca2
//System.out.println(token);
return token;
}
private String submitToken(String token) throws IOException {
// http://cafeconleche.org/books/xmljava/chapters/ch03s05.html
String url = String.format("https://%s.sharepoint.com%s", sharepointContext, loginContextPath);
URL u = new URL(url);
URLConnection uc = u.openConnection();
HttpURLConnection connection = (HttpURLConnection) uc;
connection.setDoOutput(true);
connection.setDoInput(true);
connection.setRequestMethod("POST");
connection.addRequestProperty("Accept", "application/x-www-form-urlencoded");
connection.addRequestProperty("User-Agent", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0)");
// http://stackoverflow.com/questions/12294274/mobile-app-for-sharepoint/12295224#12295224
connection.addRequestProperty("Content-Type", "text/xml; charset=utf-8");
connection.setInstanceFollowRedirects(false);
OutputStream out = connection.getOutputStream();
Writer wout = new OutputStreamWriter(out);
wout.write(token);
wout.flush();
wout.close();
InputStream in = connection.getInputStream();
//http://www.exampledepot.com/egs/java.net/GetHeaders.html
String setCookieRtFa = null;
String setCookieFedAuth = null;
for (int i=0; ; i++) {
String headerName = connection.getHeaderFieldKey(i);
String headerValue = connection.getHeaderField(i);
//System.out.println("header: " + headerName + " : " + headerValue);
if (headerName == null && headerValue == null) {
// No more headers
break;
}
if (headerName == null) {
// The header value contains the server's HTTP version
}
if (headerName != null) {
if (headerName.equals("Set-Cookie")) {
if (setCookieRtFa == null) {
setCookieRtFa = headerValue;
} else {
int t = 0;
if (headerValue.equals("RpsContextCookie=; path=/")) t = 1;
if (t == 0) {
setCookieFedAuth = headerValue;
}
}
}
}
}
String cookieContainer = setCookieRtFa.split("\\;")[0] + ";" + setCookieFedAuth.split("\\;")[0];
in.close();
return cookieContainer;
}
}
LoginManagerSharePoint loginManagerSharePoint = new LoginManagerSharePoint();
String cookieContainer = loginManagerSharePoint.getCookie();
HttpClient httpClient = HttpClientBuilder.create().build();
HttpGet httpGet = new HttpGet(URL_FILE);
httpGet.addHeader("Cookie", cookieContainer);
httpGet.addHeader("User-Agent", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0)");
HttpResponse response = httpClient.execute(httpGet);
InputStream inputStream = response.getEntity().getContent();

Categories

Resources