Develop Reference

AWS ElasticSearch/OpenSearch not connecting from java

Im using a simple example from amazon aws site to connect to opensearch index.
This is the example source https://docs.aws.amazon.com/opensearch-service/latest/developerguide/request-signing.html#request-signing-java.
The health status of my node is yellow and its open
yellow open my-index
The error message
Exception in thread "main" java.net.ConnectException
at org.elasticsearch.client.RestClient$SyncResponseListener.get(RestClient.java:943)
at org.elasticsearch.client.RestClient.performRequest(RestClient.java:227)
at org.elasticsearch.client.RestHighLevelClient.performRequest(RestHighLevelClient.java:1256)
at org.elasticsearch.client.RestHighLevelClient.performRequestAndParseEntity(RestHighLevelClient.java:1231)
at org.elasticsearch.client.RestHighLevelClient.index(RestHighLevelClient.java:587)
at com.amazonaws.lambda.demo.AWSElasticsearchServiceClient.main(AWSElasticsearchServiceClient.java:41)
Caused by: java.net.ConnectException
at org.apache.http.nio.pool.RouteSpecificPool.timeout(RouteSpecificPool.java:168)
at org.apache.http.nio.pool.AbstractNIOConnPool.requestTimeout(AbstractNIOConnPool.java:561)
at org.apache.http.nio.pool.AbstractNIOConnPool$InternalSessionRequestCallback.timeout(AbstractNIOConnPool.java:822)
at org.apache.http.impl.nio.reactor.SessionRequestImpl.timeout(SessionRequestImpl.java:183)
at org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor.processTimeouts(DefaultConnectingIOReactor.java:210)
at org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor.processEvents(DefaultConnectingIOReactor.java:155)
at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor.execute(AbstractMultiworkerIOReactor.java:348)
at org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager.execute(PoolingNHttpClientConnectionManager.java:192)
at org.apache.http.impl.nio.client.CloseableHttpAsyncClientBase$1.run(CloseableHttpAsyncClientBase.java:64)
at java.lang.Thread.run(Unknown Source) ```
private static String region = "us-west-1";
private static String domainEndpoint = "<my-index...amazon.com>"; // e.g. https://search-mydomain.us-west-1.es.amazonaws.com
private static String index = "my-index";
private static String type = "_doc";
private static String id = "1";
static final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
``` public static void main(String[] args) throws IOException {
RestHighLevelClient searchClient = searchClient(serviceName, region);
// Create the document as a hash map
Map<String, Object> document = new HashMap<>();
document.put("title", "Walk the Line");
document.put("director", "James Mangold");
document.put("year", "2005");
// Form the indexing request, send it, and print the response
IndexRequest request = new IndexRequest(index, type, id).source(document);
IndexResponse response = searchClient.index(request, RequestOptions.DEFAULT);
System.out.println(response.toString());
}
// Adds the interceptor to the OpenSearch REST client
public static RestHighLevelClient searchClient(String serviceName, String region) {
AWS4Signer signer = new AWS4Signer();
signer.setServiceName(serviceName);
signer.setRegionName(region);
HttpRequestInterceptor interceptor = new AWSRequestSigningApacheInterceptor(serviceName, signer, credentialsProvider);
return new RestHighLevelClient(RestClient.builder(HttpHost.create(domainEndpoint)).setHttpClientConfigCallback(hacb -> hacb.addInterceptorLast(interceptor)));
}

Try this example. I tried the same and it did work well for me. I did not bother doing anything in regards to the cert as I had followed AWS demo examples to create the domain.
Hopefully this is what you are looking for...

Error with Spring AppRoleAuthentication - URI is not absolute

I'm trying to retrieve secrets from vault using the AppRole authentication. But I get the error :
java.lang.IllegalArgumentException: URI is not absolute
What I do is create a vaultEndpoint then depending on the method choosen I use token authentication or AppRole authentication. There's no issue with the token authentication, however whenever I try to retrive a secret or even get the vaultToken to login with AppRole the URI is not absolute error occurs.
I 've seen in https://docs.oracle.com/javase/8/docs/api/java/net/URI.html that an URI is absolute when it specifies a scheme otherwise it is relative. But I think that my URI is specifing a scheme.
So I'm a bit lost here.
Does anyone know what I am doing wrong ? Or why I get this error ?
I use spring-vault-core-2.2.0.RELEASE
Here's my code :
VaultEndpoint ep = VaultEndpoint.create(host, portInt);
if (scheme != null) {
ep.setScheme(scheme);
}
if (authMethod.equals("token")) {
vaultTemplate = new VaultTemplate(ep, new TokenAuthentication(token));
} else if (authMethod.equals("appRole")) {
RestOperations restOperations = VaultClients.createRestTemplate();
AppRoleAuthenticationOptions options = AppRoleAuthenticationOptions.builder()
.roleId(AppRoleAuthenticationOptions.RoleId.provided(roleId))
.secretId(AppRoleAuthenticationOptions.SecretId.wrapped(VaultToken.of(secretId))).build();
vaultTemplate = new VaultTemplate(ep, new AppRoleAuthentication(options, restOperations));
}
}
I have the same error if I try to get the vaultToken :
RestOperations restOperations = VaultClients.createRestTemplate();
AppRoleAuthenticationOptions options = AppRoleAuthenticationOptions.builder()
.roleId(AppRoleAuthenticationOptions.RoleId.provided(roleId))
.secretId(AppRoleAuthenticationOptions.SecretId.wrapped(VaultToken.of(uncryptedSecretId))).build();
AppRoleAuthentication appRoleAuth = new AppRoleAuthentication(options, restOperations);
VaultToken appRoleToken = appRoleAuth.login();
Here is the error :
java.lang.IllegalArgumentException: URI is not absolute
at java.net.URI.toURL(Unknown Source)
at org.springframework.http.client.SimpleClientHttpRequestFactory.createRequest(SimpleClientHttpRequestFactory.java:145)
at org.springframework.http.client.InterceptingClientHttpRequest$InterceptingRequestExecution.execute(InterceptingClientHttpRequest.java:98)
at org.springframework.vault.client.VaultClients.lambda$createRestTemplate$0(VaultClients.java:128)
at org.springframework.http.client.InterceptingClientHttpRequest$InterceptingRequestExecution.execute(InterceptingClientHttpRequest.java:93)
at org.springframework.http.client.InterceptingClientHttpRequest.executeInternal(InterceptingClientHttpRequest.java:77)
at org.springframework.http.client.AbstractBufferingClientHttpRequest.executeInternal(AbstractBufferingClientHttpRequest.java:48)
at org.springframework.http.client.AbstractClientHttpRequest.execute(AbstractClientHttpRequest.java:53)
at org.springframework.web.client.RestTemplate.doExecute(RestTemplate.java:742)
at org.springframework.web.client.RestTemplate.execute(RestTemplate.java:677)
at org.springframework.web.client.RestTemplate.exchange(RestTemplate.java:586)
at org.springframework.vault.authentication.AppRoleAuthentication.getSecretId(AppRoleAuthentication.java:305)
at org.springframework.vault.authentication.AppRoleAuthentication.getAppRoleLoginBody(AppRoleAuthentication.java:344)
at org.springframework.vault.authentication.AppRoleAuthentication.createTokenUsingAppRole(AppRoleAuthentication.java:201)
at org.springframework.vault.authentication.AppRoleAuthentication.login(AppRoleAuthentication.java:191)

After further investigation, the issue was how I instanciated the restTemplate.
I added the spring context library to my project and implemented the AbstractVaultConfiguration class. This class contains a restOperations() function that solved my problem.
This is how I solved the issue :
public class AppRoleAuthenticationService extends AbstractVaultConfiguration {
private String roleId;
private String secretId;
private String host;
private String scheme;
private String port;
public AppRoleAuthenticationService(String roleId, String secretId, String host, String scheme, String port) {
this.roleId = roleId;
this.secretId = secretId;
this.host = host;
this.scheme = scheme;
this.port = port;
}
#Override
public VaultEndpoint vaultEndpoint() {
int portInt = Integer.parseInt(port);
VaultEndpoint ep = VaultEndpoint.create(host, portInt);
if (scheme != null) {
ep.setScheme(scheme);
}
return ep;
}
#Override
public ClientAuthentication clientAuthentication() {
AppRoleAuthenticationOptions options = AppRoleAuthenticationOptions.builder()
.roleId(AppRoleAuthenticationOptions.RoleId.provided(roleId))
.secretId(AppRoleAuthenticationOptions.SecretId.provided(secretId)).build();
return new AppRoleAuthentication(options, restOperations());
}
}
And then just use this class :
AppRoleAuthenticationService appRoleAuth = new AppRoleAuthenticationService(roleId,
uncryptedSecretId, host, scheme, port);
VaultEndpoint vaultEp = appRoleAuth.vaultEndpoint();
ClientAuthentication auth = appRoleAuth.clientAuthentication();
vaultTemplate = new VaultTemplate(vaultEp, auth);

Retrieve all Managed Devices using Java (with Microsoft Graph API SDK)

I would like to retrieve all devices managed by Intune (managed devices) using the Microsoft Graph Java SDK. I have created the app in Microsoft Azure and given the appropriate API permissions:
API Permissions
The following code creates a graphClient object and a method that retrieves all managed devices.
#Service
public class AzureServiceDefault implements AzureService
{
private static final String CLIENT_ID = "XXXXXXXXXXXXXXXXXXXXXXXX";
private static final List<String> SCOPES = Arrays.asList(new String[]{"https://graph.microsoft.com/.default"});
private static final String TENANT = "XXXXXXXXXXXXXXXXXXXXXXXX";
private static final String CLIENT_SECRET = "XXXXXXXXXXXXXXXXXXXXXXXX";
ClientCredentialProvider authProvider = new ClientCredentialProvider(CLIENT_ID, SCOPES, CLIENT_SECRET, TENANT, NationalCloud.Global);
IGraphServiceClient graphClient;
public AzureServiceDefault()
{
graphClient = GraphServiceClient.builder().authenticationProvider(authProvider).buildClient();
}
#Override
public List<IntuneDevice> getManagedDevices()
{
IManagedDeviceCollectionRequestBuilder managedDeviceRequestBuilder;
IDeviceManagementRequestBuilder builder = graphClient.deviceManagement();
IDeviceManagementRequest managedDevicesRequest = builder.buildRequest();
List<ManagedDevice> managedDevices = new ArrayList<>();
List<IntuneDevice> allManagedDevices = new ArrayList<>();
do {
try {
DeviceManagement deviceManagement = managedDevicesRequest.get();
ManagedDeviceCollectionPage managedDevicesCollectionPage = deviceManagement.managedDevices;
//Process items in the response
managedDevices.addAll(managedDevicesCollectionPage.getCurrentPage());
managedDevices.stream().forEach((device) -> allManagedDevices.add(new IntuneDevice(device.id,
device.userId,
device.deviceName,
device.managedDeviceOwnerType.toString(),
device.operatingSystem,
device.osVersion,
device.complianceState.toString(),
device.azureADRegistered,
device.azureADDeviceId,
device.userPrincipalName,
device.model,
device.manufacturer,
device.serialNumber)));
//Build the request for the next page, if there is one
managedDeviceRequestBuilder = managedDevicesCollectionPage.getNextPage();
if (managedDeviceRequestBuilder == null)
{
managedDevicesRequest = null;
}
else
{
managedDevicesRequest = (IDeviceManagementRequest) managedDeviceRequestBuilder.buildRequest();
}
}
catch(ClientException ex)
{
ex.printStackTrace();
managedDevicesRequest = null;
}
} while (managedDevicesRequest != null);
return allManagedDevices;
}
}
The problem is that the variable managedDevices turns out to be null and this is the error message:
SEVERE: Servlet.service() for servlet [dispatcher] in context with path [/] threw exception [Request processing failed; nested exception is java.lang.NullPointerException: Cannot invoke "com.microsoft.graph.requests.extensions.ManagedDeviceCollectionPage.getCurrentPage()" because "managedDevicesCollectionPage" is null] with root cause
java.lang.NullPointerException: Cannot invoke "com.microsoft.graph.requests.extensions.ManagedDeviceCollectionPage.getCurrentPage()" because "managedDevicesCollectionPage" is null
What do I need to change to make this code work? I am succesfully able to retrieve all users in Azure AD, but I am having difficulties getting data from Intune/Endpoint Manager. Do I need to make changes to the SCOPES?
It should be possible to retrieve all managed devices as the REST API for it is https://graph.microsoft.com/v1.0/deviceManagement/managedDevices
Thanks for your help

This MS Graph API does not support application permissions, so you couldn't list managedDevices with ClientCredentialProvider. ClientCredentialProvider is based on client credential flow that requires application permission.
You could use AuthorizationCodeProvider to get the list. And follow this to get AUTHORIZATION_CODE first.
String CLIENT_ID = "xxxxxx";
List<String> SCOPES = Arrays.asList(new String[] { "https://graph.microsoft.com/.default" });
String CLIENT_SECRET = "xxxxxx";
String TENANT = "xxxxxx";
String AUTHORIZATION_CODE = "";
String REDIRECT_URL = "xxxxxx";
AuthorizationCodeProvider authProvider = new AuthorizationCodeProvider(CLIENT_ID, SCOPES, AUTHORIZATION_CODE,
REDIRECT_URL, NationalCloud.Global, TENANT, CLIENT_SECRET);
IGraphServiceClient graphClient = GraphServiceClient.builder().authenticationProvider(authProvider).buildClient();
IManagedDeviceCollectionPage managedDeviceCollectionPage = graphClient.deviceManagement().managedDevices().buildRequest().get();
List<ManagedDevice> managedDeviceList = managedDeviceCollectionPage.getCurrentPage();

Retrieve a list of a given user's tweets using Twitter API 1.1 and Retrofit

I'm trying to obtain a list of a user's tweets and I've run into some trouble when trying to authenticate my call to the API. I currently get a 401 when executing the code below:
public interface TwitterApi {
String API_URL = "https://api.twitter.com/1.1";
String CONSUMER_KEY = "<CONSUMER KEY GOES HERE>";
String CONSUMER_SECRET = "<CONSUMER SECRET GOES HERE>";
String ACCESS_TOKEN = "<ACCESS TOKEN GOES HERE>";
String ACCESS_TOKEN_SECRET = "<ACCESS TOKEN SECRET GOES HERE>";
#GET("/statuses/user_timeline.json")
List<Tweet> fetchUserTimeline(
#Query("count") final int count,
#Query("screen_name") final String screenName);
}
The following throws a 401 Authorisation error when calling fetchUserTimeline()
RetrofitHttpOAuthConsumer consumer = new RetrofitHttpOAuthConsumer(TwitterApi.CONSUMER_KEY, TwitterApi.CONSUMER_SECRET);
consumer.setTokenWithSecret(TwitterApi.ACCESS_TOKEN, TwitterApi.ACCESS_TOKEN_SECRET);
RestAdapter restAdapter = new RestAdapter.Builder()
.setEndpoint(TwitterApi.API_URL)
.setClient(new SigningOkClient(consumer))
.build();
TwitterApi twitterApi = restAdapter.create(TwitterApi.class)
tweets = twitterApi.fetchUserTimeline(2, screenName);
I've also included the relevant code from the signpost-retrofit plugin:
public class SigningOkClient extends OkClient {
private final RetrofitHttpOAuthConsumer mOAuthConsumer;
public SigningOkClient(RetrofitHttpOAuthConsumer consumer) {
mOAuthConsumer = consumer;
}
public SigningOkClient(OkHttpClient client, RetrofitHttpOAuthConsumer consumer) {
super(client);
mOAuthConsumer = consumer;
}
#Override
public Response execute(Request request) throws IOException {
Request requestToSend = request;
try {
HttpRequestAdapter signedAdapter = (HttpRequestAdapter) mOAuthConsumer.sign(request);
requestToSend = (Request) signedAdapter.unwrap();
} catch (OAuthMessageSignerException | OAuthExpectationFailedException | OAuthCommunicationException e) {
// Fail to sign, ignore
e.printStackTrace();
}
return super.execute(requestToSend);
}
}
The signpost-retrofit plugin can be found here: https://github.com/pakerfeldt/signpost-retrofit
public class RetrofitHttpOAuthConsumer extends AbstractOAuthConsumer {
private static final long serialVersionUID = 1L;
public RetrofitHttpOAuthConsumer(String consumerKey, String consumerSecret) {
super(consumerKey, consumerSecret);
}
#Override
protected HttpRequest wrap(Object request) {
if (!(request instanceof retrofit.client.Request)) {
throw new IllegalArgumentException("This consumer expects requests of type " + retrofit.client.Request.class.getCanonicalName());
}
return new HttpRequestAdapter((Request) request);
}
}
Any help here would be great. The solution doesn't have to include the use of signpost but I do want to use Retrofit. I also do not want to show the user an 'Authenticate with Twitter' screen in a WebView - I simply want to display a handful of relevant tweets as part of a detail view.

Are you certain the signpost-retrofit project works for twitter oauth? I've used twitter4j successfully in the past - and if you don't want the full library you can use their code for reference. twitter4j

How to Add new column dynamically to already existed table in bigquery..?

I created a csv file with three columns in a row..in google bigquery in created a dataset with one table with csv file ....for this i completed my java code...but now i have to add a new column to existed row dynamically in java code..?
// Main method for data print.
#SuppressWarnings({ "rawtypes", "unchecked" })
public static void main(String[] args) throws IOException, InterruptedException {
// Create a new BigQuery client authorized via OAuth 2.0 protocol
Bigquery bigquery = createAuthorizedClient();
TableRow row = new TableRow();
row.set("Column1", "Sample23");
row.set("Column2", 7);
row.set("Column3", "Sample25");
TableDataInsertAllRequest.Rows rows = new TableDataInsertAllRequest.Rows();
rows.setJson(row);
List rowList = new ArrayList();
rowList.add(rows);
TableDataInsertAllRequest content =
new TableDataInsertAllRequest().setRows(rowList);
TableDataInsertAllResponse response = bigquery.tabledata().insertAll(PROJECT_ID, DATASET_ID, TABLE_ID, content).execute();
System.out.println("Final response = " + response);
}

There are two table operations Update and Patch.
You need to use the Update command, to add new columns to your schema.
Important side notes:
order is important. If you change the ordering, it will look like an incompatible schema.
you can only add new fields at the end of the table. On the old columns, you have the option to change required to nullable.
you cannot add a required field to an existing schema.
you cannot remove old fields, once a table's schema has been specified you cannot change it without first deleting all the of the data associated with it. If you want to change a table's schema, you must specify a writeDisposition of WRITE_TRUNCATE. For more information, see the Jobs resource.
Here is an example of a curl session that adds fields to a schema. It should be relatively easy to adapt to Java. It uses auth.py from here
When using Table.Update(), you must include the full table schema again. If you don't provide an exact matching schema you can get: Provided Schema does not match Table. For example I didn't paid attention to details and in one of my update calls I didn't include an old field like created and it failed.

Actually I didn't use any jobs in my java code. I simply created a dataset with one table with a row in three columns. Now I have to add new column at java code not in csv file. I am posting my complete source code:
public class BigQueryJavaGettingStarted {
// Define required variables.
private static final String PROJECT_ID = "nvjsnsb";
private static final String DATASET_ID = "nvjjvv";
private static final String TABLE_ID = "sampledata";
private static final String CLIENTSECRETS_LOCATION = "client_secrets.json";
static GoogleClientSecrets clientSecrets = loadClientSecrets(CLIENTSECRETS_LOCATION);
// Static variables for API scope, callback URI, and HTTP/JSON functions
private static final List<String> SCOPES = Arrays.asList(BigqueryScopes.BIGQUERY);
private static final String REDIRECT_URI = "https://www.example.com/oauth2callback";
// Global instances of HTTP transport and JSON factory objects.
private static final HttpTransport TRANSPORT = new NetHttpTransport();
private static final JsonFactory JSON_FACTORY = new JacksonFactory();
private static GoogleAuthorizationCodeFlow flow = null;
// Main method for data print.
#SuppressWarnings({ "rawtypes", "unchecked" })
public static void main(String[] args) throws IOException, InterruptedException {
// Create a new BigQuery client authorized via OAuth 2.0 protocol
Bigquery bigquery = createAuthorizedClient();
TableRow row = new TableRow();
row.set("Column1", "OneMoreCol1");
row.set("Column2", 79);
row.set("Column3", "OneMoreCol2");
TableDataInsertAllRequest.Rows rows = new TableDataInsertAllRequest.Rows();
rows.setJson(row);
List rowList = new ArrayList();
rowList.add(rows);
TableDataInsertAllRequest content = new TableDataInsertAllRequest().setRows(rowList);
TableDataInsertAllResponse response = bigquery.tabledata().insertAll(PROJECT_ID, DATASET_ID, TABLE_ID, content).execute();
System.out.println("Final response = " + response);
}
// Create Authorized client.
public static Bigquery createAuthorizedClient() throws IOException {
String authorizeUrl = new GoogleAuthorizationCodeRequestUrl(
clientSecrets,
REDIRECT_URI,
SCOPES).setState("").build();
System.out.println("Paste this URL into a web browser to authorize BigQuery Access:\n" + authorizeUrl);
System.out.println("... and type the code you received here: ");
BufferedReader in = new BufferedReader(new InputStreamReader(System.in));
String authorizationCode = in.readLine();
// Exchange the auth code for an access token and refresh token
Credential credential = exchangeCode(authorizationCode);
return new Bigquery(TRANSPORT, JSON_FACTORY, credential);
}
// Exchange code method.
static Credential exchangeCode(String authorizationCode) throws IOException {
GoogleAuthorizationCodeFlow flow = getFlow();
GoogleTokenResponse response =
flow.newTokenRequest(authorizationCode).setRedirectUri(REDIRECT_URI).execute();
return flow.createAndStoreCredential(response, null);
}
// Get flow.
static GoogleAuthorizationCodeFlow getFlow() {
if (flow == null) {
HttpTransport httpTransport = new NetHttpTransport();
JacksonFactory jsonFactory = new JacksonFactory();
flow = new GoogleAuthorizationCodeFlow.Builder(httpTransport,
jsonFactory,
clientSecrets,
SCOPES)
.setAccessType("offline").setApprovalPrompt("force").build();
}
return flow;
}
// Load client secrets.
private static GoogleClientSecrets loadClientSecrets(String clientSecretsLocation) {
try {
clientSecrets = GoogleClientSecrets.load(new JacksonFactory(),
new InputStreamReader(BigQueryJavaGettingStarted.class.getResourceAsStream(clientSecretsLocation)));
} catch (Exception e) {
System.out.println("Could not load client_secrets.json");
e.printStackTrace();
}
return clientSecrets;
}
}