Develop Reference

Java HTTP POST request with JSON

After running the post request with JSON I have HTTP Error 400 Bad request.
I think the problem is in the JSON string but I do not now how to fix it.
package Curl;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.http.HttpRequest;
import java.nio.charset.StandardCharsets;
public class Http_Con {
private static HttpURLConnection connection;
public static void main(String[] args) throws IOException {
Http_Con connection = new Http_Con();
//call the getRequest method
connection.GetRequest();
//call the postRequest
connection.PostRequest();
}
public void GetRequest() {
BufferedReader reader;
String lines;
StringBuilder responseContent = new StringBuilder();
try {
//set the url
URL url = new URL("http://192.168.0.104:4041/iot/devices");
//open the connection
connection = (HttpURLConnection) url.openConnection();
//set the request method and timeout
connection.setRequestMethod("GET");
connection.setConnectTimeout(5000);
connection.setReadTimeout(5000);
//set request properties(in this case headers)
connection.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
connection.setRequestProperty("Content-Type", "application/json");
connection.setRequestProperty("Fiware-Service", " myHome");
connection.setRequestProperty("Fiware-ServicePath", "/environment");
//get the response code(200 should be OK)
int status = connection.getResponseCode();
if (status > 299) {
reader = new BufferedReader(new InputStreamReader(connection.getErrorStream()));
while ((lines = reader.readLine()) != null) {
responseContent.append(lines);
}
reader.close();
} else {
reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
while ((lines = reader.readLine()) != null) {
responseContent.append(lines);
}
reader.close();
}
//print the response content(result)
System.out.println(responseContent.toString());
} catch (Exception e) {
e.printStackTrace();
}
}
public void PostRequest() throws IOException {
//set the url
String url = "http://192.168.0.104:4041/iot/devices";
URL obj = new URL(url);
//open the connection
HttpURLConnection connection = (HttpURLConnection) obj.openConnection();
//set the method type(POST OR GET)
connection.setRequestMethod("POST");
//set hte request properties(headers etc.)
connection.setRequestProperty("Content-Type", "application/json; utf-8");
connection.setRequestProperty("Accept", "application/json");
connection.setRequestProperty("Fiware-Service"," myHome");
connection.setRequestProperty("Fiware-ServicePath","/environment");
//make sure that will be able to write content to the connection output stream
connection.setDoOutput(true);
//Json formatted input string
String jsonInputString = "{\n" +
" \"devices\": [\n" +
" {\n" +
" \"device_id\": \"sensor01\",\n" +
" \"entity_name\": \"LivingRoomSensor\",\n" +
" \"entity_type\": \"multiSensor\",\n" +
" \"attributes\": [\n" +
" { \"object_id\": \"t\", \"name\": \"Temperature\", \"type\": \"celsius\" },\n" +
" { \"object_id\": \"l\", \"name\": \"Luminosity\", \"type\": \"lumens\" }\n" +
" ]\n" +
" }\n" +
" ]\n" +
"}";
try(OutputStream os = connection.getOutputStream()) {
byte[] input = jsonInputString.getBytes(StandardCharsets.UTF_8);
os.write(input, 0, input.length);
}
try(BufferedReader br = new BufferedReader(
new InputStreamReader(connection.getInputStream(), StandardCharsets.UTF_8))) {
StringBuilder response = new StringBuilder();
String responseLine = null;
while ((responseLine = br.readLine()) != null) {
response.append(responseLine.trim());
}
//print the result
System.out.println(response.toString());
}
}
}

I Think you can use Fidder application to do something, which find Questions so fast. Due to 400 statusCode, Liking Cookie lost, params lost, request method lost .... trust think for you help！

How can I add Authorization and Date/time header in the below code?

I am creating a Java Rest api to create users on Google Duo admin. I am following the documentation https://duo.com/docs/adminapi and they need auth and date/time header. I am fairly new to java can anyone guide me?
This is the updated code which I with authentication and date/time header but now getting the below error.
UPDATED CODE
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Base64;
public class DuoAdminAPI {
public static void POSTRequest() throws IOException {
String userCredentials = "Username:Password";
String basicAuth = "Basic " + new String(Base64.getEncoder().encode(userCredentials.getBytes()));
String dateTime = OffsetDateTime.now().format(DateTimeFormatter.RFC_1123_DATE_TIME); //RFC_1123 == RFC_2822
final String POST_PARAMS = "{\n" + "\"userId\": 101,\r\n" +
" \"id\": 101,\r\n" +
" \"title\": \"Test Title\",\r\n" +
" \"body\": \"Test Body\"" + "\n}";
System.out.println(POST_PARAMS);
URL obj = new URL("https://api-e9770554.duosecurity.com");
HttpURLConnection postConnection = (HttpURLConnection) obj.openConnection();
postConnection.setRequestMethod("POST");
postConnection.setRequestProperty("Content-Type", "application/json");
postConnection.setRequestProperty("Authorization", basicAuth);
postConnection.setRequestProperty("Date", dateTime);
postConnection.setDoOutput(true);
OutputStream os = postConnection.getOutputStream();
os.write(POST_PARAMS.getBytes());
os.flush();
os.close();
int responseCode = postConnection.getResponseCode();
System.out.println("POST Response Code : " + responseCode);
System.out.println("POST Response Message : " + postConnection.getResponseMessage());
if (responseCode == HttpURLConnection.HTTP_CREATED) { //success
BufferedReader in = new BufferedReader(new InputStreamReader(
postConnection.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
// print result
System.out.println(response.toString());
} else {
System.out.println("POST NOT WORKED");
}
}
}
ERROR
{
"code": 40101,
"message": "Missing request credentials",
"stat": "FAIL"
}
Response code: 401 (Unauthorized); Time: 2022ms; Content length: 73 bytes

As I have read the documentation, the authorization is Basic:
String userCredentials = "username:password";
String basicAuth = "Basic " + new String(Base64.getEncoder().encode(userCredentials.getBytes()));
For date is this:
String dateTime = OffsetDateTime.now().format(DateTimeFormatter.RFC_1123_DATE_TIME); //RFC_1123 == RFC_2822
In their documentation they require RFC 2822 format. We already have declared RFC 1123 in DateTimeFormatter, which practically it is the same.
And this is the way you can add them in headers:
postConnection.setRequestProperty("Authorization", basicAuth);
postConnection.setRequestProperty("Date", dateTime);

how's model of post request to login gmail in java

i tried to login gmail via sending post request flowwing the code:
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package testautologingmail;
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.CookieHandler;
import java.net.CookieManager;
import java.net.HttpCookie;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import javax.net.ssl.HttpsURLConnection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
public class HttpUrlConnectionExample {
private static List<String> cookies = new ArrayList<>();;
private static HttpsURLConnection conn;
private final String USER_AGENT = "Mozilla/5.0";
public HttpUrlConnectionExample(){
}
public static void main(String[] args) throws Exception {
String url = "https://accounts.google.com/ServiceLoginAuth";
String gmail = "https://mail.google.com/mail/";
HttpUrlConnectionExample http = new HttpUrlConnectionExample();
// make sure cookies is turn on
//CookieHandler.setDefault(new CookieManager());
// 1. Send a "GET" request, so that you can extract the form's data.
String page = http.GetPageContent(url);
String postParams = http.getFormParams(page,"kiemtienol1506","vandai1507");
// 2. Construct above post's content and then send a POST request for
// authentication
http.sendPost(url, postParams);
// 3. success then go to gmail.
String result = http.GetPageContent(gmail);
System.out.println(result);
conn.disconnect();
}
private static void getcookies(String url) throws MalformedURLException, IOException{
CookieManager cookieManager = new CookieManager();
CookieHandler.setDefault(cookieManager);
URL obj = new URL(url);
conn = (HttpsURLConnection) obj.openConnection();
conn.getContent();
List<HttpCookie> cookies2 = cookieManager.getCookieStore().getCookies();
System.out.println("get cookies");
for (HttpCookie cookie : cookies2) {
System.out.println(cookie.getDomain());
System.out.println(cookie.toString());
String temp = cookie.toString();
cookies.add(temp);
}
//conn.disconnect();
}
private void sendPost(String url, String postParams) throws Exception {
URL obj = new URL(url);
conn = (HttpsURLConnection) obj.openConnection();
// Acts like a browser
conn.setUseCaches(false);
conn.setRequestMethod("POST");
conn.setRequestProperty("Host", "accounts.google.com");
conn.setRequestProperty("User-Agent", USER_AGENT);
conn.setRequestProperty("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
System.out.println("cookies String");
for (String cookie : cookies) {
//String[] tmp = cookie.split(":");
conn.addRequestProperty("Cookie", cookie.split(";", 1)[0]);
System.out.println(cookie.split(";", 1)[0]);
}
conn.setRequestProperty("Connection", "keep-alive");
conn.setRequestProperty("Referer", "https://accounts.google.com/ServiceLoginAuth");
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
conn.setRequestProperty("Content-Length", Integer.toString(postParams.length()));
conn.setDoOutput(true);
conn.setDoInput(true);
// Send post request
DataOutputStream wr = new DataOutputStream(conn.getOutputStream());
wr.writeBytes(postParams);
wr.flush();
wr.close();
int responseCode = conn.getResponseCode();
System.out.println("\nSending 'POST' request to URL : " + url);
System.out.println("Post parameters : " + postParams);
System.out.println("Response Code : " + responseCode);
BufferedReader in =
new BufferedReader(new InputStreamReader(conn.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
//System.out.println(response.toString());
//conn.disconnect();
}
private String GetPageContent(String url) throws Exception {
URL obj = new URL(url);
conn = (HttpsURLConnection) obj.openConnection();
// default is GET
conn.setRequestMethod("GET");
conn.setUseCaches(false);
// act like a browser
conn.setRequestProperty("User-Agent", USER_AGENT);
conn.setRequestProperty("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
if (cookies != null) {
for (String cookie : cookies) {
conn.addRequestProperty("Cookie", cookie.split(";", 1)[0]);
}
}
int responseCode = conn.getResponseCode();
System.out.println("\nSending 'GET' request to URL : " + url);
System.out.println("Response Code : " + responseCode);
BufferedReader in =
new BufferedReader(new InputStreamReader(conn.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
// Get the response cookies
//setCookies(conn.getHeaderFields().get("Set-Cookie"));
getcookies(url);
return response.toString();
}
public String getFormParams(String html, String username, String password)
throws UnsupportedEncodingException {
System.out.println("Extracting form's data...");
Document doc = Jsoup.parse(html);
// Google form id
Element loginform = doc.getElementById("gaia_loginform");
Elements inputElements = loginform.getElementsByTag("input");
List<String> paramList = new ArrayList<String>();
for (Element inputElement : inputElements) {
String key = inputElement.attr("name");
String value = inputElement.attr("value");
String id = inputElement.attr("id");
if (key.equals("Email"))
{
value = username;
paramList.add(key + "=" + URLEncoder.encode(value, "UTF-8"));
}
else if (key.equals("Passwd")){
paramList.add(key + "=" + URLEncoder.encode(value, "UTF-8"));
value = password;
}
else if(id.equals("Passwd-hidden")){
value = password;
paramList.add("Passwd=" + URLEncoder.encode(value, "UTF-8"));
}else
paramList.add(key + "=" + URLEncoder.encode(value, "UTF-8"));
}
// build parameters list
StringBuilder result = new StringBuilder();
for (String param : paramList) {
if (result.length() == 0) {
result.append(param);
} else {
result.append("&" + param);
}
}
return result.toString();
}
public List<String> getCookies() {
return cookies;
}
public void setCookies(List<String> cookies) {
this.cookies = cookies;
}
}
i also try to get cookies flow this line setCookies(conn.getHeaderFields().get("Set-Cookie")); but it's always return null String so i have to make getCookies() function.
i always receive status 200 OK. So is there any support to help me or another code can do this task.
Thanks in Advance.

Jsoup parsing google results

I want to grab all the links and thumbnails wherever given from a particular google search. Here is my code.
package com.esocial.util;
import java.io.IOException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
public class ListLinks {
public static void main(String[] args) throws IOException {
String url = "https://www.google.co.in/webhp?sourceid=chrome-instant&rlz=1C1CHWA_enIN609IN609&ion=1&espv=2&ie=UTF-8#q=thermodynamics%20cbse";
System.out.println("Fetching : "+url+"\n\n");
Document doc = Jsoup.connect(url).userAgent("Mozilla").get();
Elements div = doc.select("div.srg");
for(Element di : div)
{
Elements lists = di.select("li.g");
for(Element list : lists)
{
Element anc = list.select("a").first();
Element img = list.select("img").first();
System.out.println("\nLink : "+anc.attr("href")+"\nImage Link : "+img.attr("src")+"\n------------------------------------------\n");
}
}
}
}
But this code is not running properly and does not display results. I don't understand what the problem is.

I have also worked on to grab the search result.
To get html page . I requested a post REST call.
// HTTP GET request
private void sendGet(String query, GoogleSearchCallback googleSearchCallback) throws Exception {
ArrayList<GoogleSearchResult> googleSearchResults = new ArrayList<>();
String url = "https://www.google.co.in/search?q=" + query + "#q=" + query + "course";
URL obj = new URL(url);
HttpURLConnection con = (HttpURLConnection) obj.openConnection();
// optional default is GET
con.setRequestMethod("GET");
con.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64)");
int responseCode = con.getResponseCode();
System.out.println("\nSending 'GET' request to URL : " + url);
System.out.println("Response Code : " + responseCode);
BufferedReader in = new BufferedReader(new InputStreamReader(
con.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
// print result
System.out.println(response.toString());
Document document = Jsoup.parse(response.toString());
Elements links = document.select("a[href]");
for (Element link : links) {
String temp = link.attr("href");
if (temp.startsWith("/url?q=")) {
Log.i(TAG, "" + temp.replace("/url?q=", "") + "");
URL tempUrl = new URL(temp.replace("/url?q=", ""));
String path = tempUrl.getFile().substring(0, tempUrl.getFile().lastIndexOf('/'));
String base = tempUrl.getProtocol() + "://" + tempUrl.getHost() + path;
Log.i(TAG, "" + base);
Log.e(TAG, getDomainName(temp));
GoogleSearchResult googleSearchResult = new GoogleSearchResult();
googleSearchResult.setResultLink(base);
googleSearchResult.setResultTitle(link.text());
googleSearchResults.add(googleSearchResult);
}
}
googleSearchCallback.onGetGoogleResultSuccess(googleSearchResults);
}
private String getDomainName(String url) {
String domainName = "";
matcher = patternDomainName.matcher(url);
if (matcher.find()) {
domainName = matcher.group(0).toLowerCase().trim();
}
return domainName;
}
private static Pattern patternDomainName;
private Matcher matcher;
private static final String DOMAIN_NAME_PATTERN = "^(http(s)?|ftp|file)://[-a-zA-Z0-9+&##/%?=~_|!:,.;]*[-a-zA-Z0-9+&##/%=~_|]";
static {
patternDomainName = Pattern.compile(DOMAIN_NAME_PATTERN);
}

Writing a java program to login to HTTP PHP website

Have a good day! I am new to java. I wrote a program in java to login to a HTTP PHP website. When I run the program it doesn't get logged in. It just displays the content of the initial login page itself and not the page that comes after logged in. I don't know what my mistake is. I have searched a lot in the web and in this website as well. But no luck for me. :(
This is my program. Please let me know if there is any error in it.
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.CookieHandler;
import java.net.CookieManager;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import java.net.HttpURLConnection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
public class HttpUrlConnectionExample1 {
private List<String> cookies;
private HttpURLConnection conn;
private final String USER_AGENT = "Mozilla/5.0";
public static void main(String[] args) throws Exception {
String url = "http://www.iwhatgroups.com/log%20in.php";
String email = "http://www.iwhatgroups.com";
HttpUrlConnectionExample1 http = new HttpUrlConnectionExample1();
// make sure cookies is turn on
CookieHandler.setDefault(new CookieManager());
// 1. Send a "GET" request, so that you can extract the form's data.
String page = http.GetPageContent(url);
String postParams = http.getFormParams(page, "MyUserId", "MyPassWord");
// 2. Construct above post's content and then send a POST request for
// authentication
http.sendPost(url, postParams);
// 3. success then go to email.
String result = http.GetPageContent(email);
System.out.println(result);
}
private void sendPost(String url, String postParams) throws Exception {
URL obj = new URL(url);
conn = (HttpURLConnection) obj.openConnection();
// Acts like a browser
conn.setUseCaches(false);
conn.setRequestMethod("POST");
conn.setRequestProperty("Host", "iwhatgroups.com");
conn.setRequestProperty("User-Agent", USER_AGENT);
// conn.setRequestProperty("Accept",
// "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
// conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
// for (String cookie : this.cookies) {
// conn.addRequestProperty("Cookie", cookie.split(";", 1)[0]);
// }
// conn.setRequestProperty("Connection", "keep-alive");
conn.setRequestProperty("Referer", "http://iwhatgroups.com");
// conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
// conn.setRequestProperty("Content-Length", Integer.toString(postParams.length()));
conn.setDoOutput(true);
conn.setDoInput(true);
// Send post request
DataOutputStream wr = new DataOutputStream(conn.getOutputStream());
wr.writeBytes(postParams);
wr.flush();
wr.close();
int responseCode = conn.getResponseCode();
System.out.println("\nSending 'POST' request to URL : " + url);
System.out.println("Post parameters : " + postParams);
System.out.println("Response Code : " + responseCode);
BufferedReader in =
new BufferedReader(new InputStreamReader(conn.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
// System.out.println(response.toString());
}
private String GetPageContent(String url) throws Exception {
URL obj = new URL(url);
conn = (HttpURLConnection) obj.openConnection();
// default is GET
conn.setRequestMethod("GET");
conn.setUseCaches(false);
// act like a browser
conn.setRequestProperty("User-Agent", USER_AGENT);
conn.setRequestProperty("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
if (cookies != null) {
for (String cookie : this.cookies) {
conn.addRequestProperty("Cookie", cookie.split(";", 1)[0]);
}
}
int responseCode = conn.getResponseCode();
System.out.println("\nSending 'GET' request to URL : " + url);
System.out.println("Response Code : " + responseCode);
BufferedReader in =
new BufferedReader(new InputStreamReader(conn.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
System.out.println(inputLine);
}
in.close();
// Get the response cookies
setCookies(conn.getHeaderFields().get("Set-Cookie"));
return response.toString();
}
public String getFormParams(String html, String username, String password)
throws UnsupportedEncodingException {
System.out.println("Extracting form's data...");
Document doc = Jsoup.parse(html);
// Form id
Element loginform = doc.getElementById("mainBody");
Elements inputElements = loginform.getElementsByTag("div");
List<String> paramList = new ArrayList<String>();
for (Element inputElement : inputElements) {
String key = inputElement.attr("name");
String value = inputElement.attr("value");
if (key.equals("inputEmail"))
value = username;
else if (key.equals("inputPassword"))
value = password;
paramList.add(key + "=" + URLEncoder.encode(value, "UTF-8"));
}
// build parameters list
StringBuilder result = new StringBuilder();
for (String param : paramList) {
if (result.length() == 0) {
result.append(param);
} else {
result.append("&" + param);
}
}
return result.toString();
}
public List<String> getCookies() {
return cookies;
}
public void setCookies(List<String> cookies) {
this.cookies = cookies;
}
}
Thank you all for reading or answering this.
NOTE:If I uncomment setRequestProperty it throws NullPointerException. Otherwise (I mean if I comment them) it gives the login page. Please help me out if possible.
Thanks again.

The problems here might be a lot.
- No Redirect handling in GET call
- Wrong URL for POST call
- Missing header parameters.
I would suggest you to use Firebug and debug with a browser a login performed by it.Sometimes the login is handled using JavaScript and the real POST URL might be different from the URL of the login page.