Develop Reference

EMR Cluster creation using Java

I am trying to create an EMR Cluster using Java. I have created the jar file and put that into a lambda function. I'm calling the lambda from AWS Step functions. I created the maven package including the AWS JAVA SDK dependencies and also imported all the packages
import java.io.IOException;
import com.amazonaws.auth.*;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.auth.PropertiesCredentials;
import com.amazonaws.services.elasticmapreduce.*;
import com.amazonaws.services.elasticmapreduce.model.AddJobFlowStepsRequest;
import com.amazonaws.services.elasticmapreduce.model.AddJobFlowStepsResult;
import com.amazonaws.services.elasticmapreduce.model.RunJobFlowRequest;
import com.amazonaws.services.elasticmapreduce.model.*;
import com.amazonaws.services.elasticmapreduce.model.HadoopJarStepConfig;
import com.amazonaws.services.elasticmapreduce.model.StepConfig;
import com.amazonaws.services.elasticmapreduce.util.StepFactory;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
public class CreateCluster {
public static void main(String[] args) {
AWSCredentials credentials = new BasicAWSCredentials("access key", "secret key");
// myApp={[Hadoop]};
AmazonElasticMapReduceClient emr = new AmazonElasticMapReduceClient(credentials);
String COMMAND_RUNNER = "command-runner.jar";
String DEBUGGING_COMMAND = "state-pusher-script";
String DEBUGGING_NAME = "Setup Hadoop Debugging";
StepFactory stepFactory = new StepFactory();
StepConfig enabledebugging = new StepConfig()
.withName(DEBUGGING_NAME)
.withActionOnFailure(ActionOnFailure.TERMINATE_CLUSTER)
.withHadoopJarStep(new HadoopJarStepConfig()
.withJar(COMMAND_RUNNER)
.withArgs(DEBUGGING_COMMAND));
RunJobFlowRequest request = new RunJobFlowRequest()
.withName("REMR")
.withReleaseLabel("emr-5.16.0")
.withSteps(enabledebugging)
// .withApplications(myApp)
.withLogUri("s3n://r.base.ihm/emr-log/")
.withServiceRole("service_role")
.withJobFlowRole("jobflow_role")
.withInstances(new JobFlowInstancesConfig()
.withEc2KeyName("emr")
.withEc2SubnetId("subnet-d1fbb8ee")
.withInstanceCount(3)
.withKeepJobFlowAliveWhenNoSteps(false)
.withMasterInstanceType("m4.large")
.withSlaveInstanceType("m4.large"));
RunJobFlowResult result = emr.runJobFlow(request);
}
}
but still I'm getting the error
java.lang.NoClassDefFoundError
{
"errorMessage": "Error loading class com.ihm.base.spark.CreateCluster: com/amazonaws/auth/AWSCredentials",
"errorType": "java.lang.NoClassDefFoundError"
}
Any ideas on what I am missing here?

simple example for getting google access token using oauth2.0 java

I want to fetch google analytics data using google access token, but i didn't find any example , i am getting "Exception in thread
"main" java.net.SocketException: Connection reset"
error , below is the code:-
import java.io.File;
import java.io.FileInputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.services.analyticsreporting.v4.AnalyticsReporting;
import com.google.api.services.analyticsreporting.v4.model.DateRange;
import com.google.api.services.analyticsreporting.v4.model.Dimension;
import com.google.api.services.analyticsreporting.v4.model.GetReportsRequest;
import com.google.api.services.analyticsreporting.v4.model.GetReportsResponse;
import com.google.api.services.analyticsreporting.v4.model.Metric;
import com.google.api.services.analyticsreporting.v4.model.ReportRequest;
public class AnalyticsApiTest {
public static void main(String[] args) throws Exception {
GoogleCredential credential = GoogleCredential
.fromStream(new FileInputStream(new File("d:\\clientsecretnew.json")))
.createScoped(Collections.singleton("https://www.googleapis.com/auth/analytics.readonly"));
HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport();
JsonFactory jsonFactory = JacksonFactory.getDefaultInstance();
AnalyticsReporting analyticsReporting = new AnalyticsReporting.Builder(httpTransport, jsonFactory, credential)
.setApplicationName("testing_analytics")
.build();
DateRange dateRange = new DateRange();
dateRange.setStartDate("2017-06-11");
dateRange.setEndDate("2017-06-13");
// Create the Metrics object.
Metric sessions = new Metric()
.setExpression("ga:pageview")
.setAlias("sessions");
//Create the Dimensions object.
Dimension browser = new Dimension()
.setName("ga:browser");
// Create the ReportRequest object.
ReportRequest request = new ReportRequest()
.setViewId("ga:152411###")
.setDateRanges(Arrays.asList(dateRange))
.setDimensions(Arrays.asList(browser))
.setMetrics(Arrays.asList(sessions));
ArrayList<ReportRequest> requests = new ArrayList<ReportRequest>();
requests.add(request);
// Create the GetReportsRequest object.
GetReportsRequest getReport = new GetReportsRequest()
.setReportRequests(requests);
// Call the batchGet method.
GetReportsResponse response = analyticsReporting.reports().batchGet(getReport).execute();
}
}
Can anyone tell what is wrong in this??

I think you should try following Googles tutorials. This is the code i use ga:pageview is not a valid metric its ga:pageviews
My Code:
import com.google.api.client.extensions.java6.auth.oauth2.AuthorizationCodeInstalledApp;
import com.google.api.client.extensions.jetty.auth.oauth2.LocalServerReceiver;
import com.google.api.client.auth.oauth2.Credential;
import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow;
import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets;
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.client.util.store.DataStoreFactory;
import com.google.api.client.util.store.FileDataStoreFactory;
import com.google.api.services.analyticsreporting.v4.AnalyticsReporting;
import com.google.api.services.analyticsreporting.v4.AnalyticsReportingScopes;
import com.google.api.services.analyticsreporting.v4.model.*;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
/**
* #author Linda Lawton
*/
public class ReportingSample {
/**
* Be sure to specify the name of your application. If the application name is {#code null} or
* blank, the application will log a warning. Suggested format is "MyCompany-ProductName/1.0".
*/
private static final String APPLICATION_NAME = "DAIMTO Google Analytics Reporting V4 sample";
/** Directory to store user credentials. */
private static final java.io.File DATA_STORE_DIR =
new java.io.File(System.getProperty("user.home"), ".store/reporting_sample");
/**
* Global instance of the {#link DataStoreFactory}. The best practice is to make it a single
* globally shared instance across your application.
*/
private static FileDataStoreFactory dataStoreFactory;
/** Global instance of the HTTP transport. */
private static HttpTransport httpTransport;
/** Global instance of the JSON factory. */
private static final JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance();
private static AnalyticsReporting service ;
/** Authorizes the installed application to access user's protected data. */
private static Credential authorize() throws Exception {
// load client secrets
GoogleClientSecrets clientSecrets = GoogleClientSecrets.load(JSON_FACTORY,
new InputStreamReader(ReportingSample.class.getResourceAsStream("/client_secrets.json")));
// set up authorization code flow
GoogleAuthorizationCodeFlow flow = new GoogleAuthorizationCodeFlow.Builder(
httpTransport, JSON_FACTORY, clientSecrets,
Collections.singleton(AnalyticsReportingScopes.ANALYTICS_READONLY)).setDataStoreFactory(
dataStoreFactory).build();
// authorize
return new AuthorizationCodeInstalledApp(flow, new LocalServerReceiver()).authorize("user");
}
public static void main(String[] args) {
try {
httpTransport = GoogleNetHttpTransport.newTrustedTransport();
dataStoreFactory = new FileDataStoreFactory(DATA_STORE_DIR);
// authorization
Credential credential = authorize();
// set up global Analytics instance
service = new AnalyticsReporting.Builder(httpTransport, JSON_FACTORY, credential).setApplicationName(
APPLICATION_NAME).build();
// run commands
fetchData(service);
// success!
return;
} catch (IOException e) {
System.err.println(e.getMessage());
} catch (Throwable t) {
t.printStackTrace();
}
System.exit(1);
}
public static void fetchData(AnalyticsReporting service) throws IOException{
DateRange dateRange = new DateRange();
dateRange.setStartDate("2017-06-11");
dateRange.setEndDate("2017-06-13");
// Create the Metrics object.
Metric sessions = new Metric()
.setExpression("ga:sessions")
.setAlias("sessions");
//Create the Dimensions object.
Dimension browser = new Dimension()
.setName("ga:browser");
// Create the ReportRequest object.
ReportRequest request = new ReportRequest()
.setViewId("ga:78110423")
.setDateRanges(Arrays.asList(dateRange))
.setDimensions(Arrays.asList(browser))
.setMetrics(Arrays.asList(sessions));
ArrayList<ReportRequest> requests = new ArrayList<ReportRequest>();
requests.add(request);
// Create the GetReportsRequest object.
GetReportsRequest getReport = new GetReportsRequest()
.setReportRequests(requests);
// Call the batchGet method.
GetReportsResponse response = service.reports().batchGet(getReport).execute();
}
}

How to authenticate my Picasaweb Albums App with Oauth2

So I followed this tutorial to make a basic Picasawebalbums Api app. I got the tutorial here https://developers.google.com/picasa-web/docs/2.0/developers_guide_java
I had problems described here Create PicasawebService and here Error while using PicasawebService with GAE and here upload image to Picasa using picasawebservice in android
I solved the classNotFoundException and now have an authenticationException. I found some questions about that but none of them helped.
stack trace
Exception in thread "main" com.google.gdata.util.AuthenticationException: Error authenticating (check service name)
at com.google.gdata.client.GoogleAuthTokenFactory.getAuthException(GoogleAuthTokenFactory.java:688)
at com.google.gdata.client.GoogleAuthTokenFactory.getAuthToken(GoogleAuthTokenFactory.java:560)
at com.google.gdata.client.GoogleAuthTokenFactory.setUserCredentials(GoogleAuthTokenFactory.java:397)
at com.google.gdata.client.GoogleService.setUserCredentials(GoogleService.java:364)
at com.google.gdata.client.GoogleService.setUserCredentials(GoogleService.java:319)
at com.google.gdata.client.GoogleService.setUserCredentials(GoogleService.java:303)
at duh.Test.main(Test.java:37)
Here is my code
public static void main(String args[]) throws IOException, ServiceException, AuthenticationException{
PicasawebService myService = new PicasawebService(APP_NAME);
myService.setConnectTimeout(60000);
myService.setReadTimeout(60000);
myService.setUserCredentials(EMAIL, PASSWORD);
URL feedUrl = new URL("https://picasaweb.google.com/data/feed/api/user/samdup123?kind=album");
UserFeed myUserFeed = myService.getFeed(feedUrl, UserFeed.class);
for(AlbumEntry myAlbum : myUserFeed.getAlbumEntries()){
System.out.println(myAlbum.getTitle().getPlainText());
}
}
}
And my imports
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import com.google.gdata.client.*;
import com.google.gdata.client.photos.*;
import com.google.gdata.client.photos.PicasawebService;
import com.google.gdata.data.*;
import com.google.gdata.data.media.*;
import com.google.gdata.data.photos.*;
import com.google.gdata.util.AuthenticationException;
import com.google.gdata.util.ServiceException;
Line 37 from the stack trace points to myService.setUserCredentials(EMAIL, PASSWORD);
It seems like I need to drop this code entirely and use OAuth2 to get authenticated. I have read about that but it is very complicated reading and I have no idea what to use for my application. I found this code
SPREADSHEET_FEED_URL = new URL("https://spreadsheets.google.com/feeds/spreadsheets/private/full");
File p12 = new File("D:\\cgi\\RemedyClientTest\\src\\main\\resources\\RemedyClient-b04b4c2d956c.p12");
factory = FeedURLFactory.getDefault();
HttpTransport httpTransport = new NetHttpTransport();
JacksonFactory jsonFactory = new JacksonFactory();
String [] SCOPESArray= {"https://spreadsheets.google.com/feeds", "https://spreadsheets.google.com/feeds/spreadsheets/private/full", "https://docs.google.com/feeds"};
final List SCOPES = Arrays.asList(SCOPESArray);
GoogleCredential credential = new GoogleCredential.Builder()
.setTransport(httpTransport)
.setJsonFactory(jsonFactory)
.setServiceAccountId("1088335521358-ijasu5h7kauqbja6hih91l2c23amv7ft#developer.gserviceaccount.com")
.setServiceAccountScopes(SCOPES)
.setServiceAccountPrivateKeyFromP12File(p12)
.build();
service = new SpreadsheetService("Main");
service.setOAuth2Credentials(credential);
But I don't know how to 'translate' that to PicasawebAlbums. I have tried but fell short because I have no idea what jar files to import.
Am i right to assume that my problem is that I need to use Oauth2? Is there anything else I should use? If someone has a working application can you show me all of the imports you have because some of mine seem to be outdated and replaced by others and its all very confusing.

Access to Google Drive Spreadsheet from remote Server

I have client/server application. The server side must access a Spreadsheet on my own Google Drive.I make a test class to try to access to the spreadsheet list:
package com.eng.app;
import java.io.File;
import java.io.IOException;
import java.net.Authenticator;
import java.net.PasswordAuthentication;
import java.net.URISyntaxException;
import java.net.URL;
import java.security.GeneralSecurityException;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
import com.google.api.client.auth.oauth2.Credential;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.gdata.client.spreadsheet.SpreadsheetService;
import com.google.gdata.data.spreadsheet.SpreadsheetEntry;
import com.google.gdata.data.spreadsheet.SpreadsheetFeed;
import com.google.gdata.util.ServiceException;
public class TestGoogleCredential {
public static void main(String[] args) throws IOException,
InterruptedException, GeneralSecurityException, URISyntaxException, ServiceException {
List<String> SCOPES = Arrays
.asList("https://spreadsheets.google.com/feeds");
final HttpTransport TRANSPORT = new NetHttpTransport();
final JsonFactory JSON_FACTORY = new JacksonFactory();
File cert = new File(Credential.class.getClassLoader()
.getResource("<the_file>.p12").toURI());
HttpTransport transport = new NetHttpTransport();
JacksonFactory jsonFactory = new JacksonFactory();
GoogleCredential credential = new GoogleCredential.Builder()
.setTransport(transport)
.setJsonFactory(jsonFactory)
.setServiceAccountId(<email-address>)
.setServiceAccountScopes(SCOPES)
.setServiceAccountPrivateKeyFromP12File(cert).build();
SpreadsheetService service = new SpreadsheetService("Inertia");
service.setOAuth2Credentials(credential);
String urlString = "https://spreadsheets.google.com/feeds/spreadsheets/private/full";
URL SPREADSHEET_FEED_URL = new URL(urlString);
// istantiate the feed to the spreadsheets
SpreadsheetFeed feed = service.getFeed(SPREADSHEET_FEED_URL,SpreadsheetFeed.class);
// the list of all spreadsheets
List<SpreadsheetEntry> spreadsheets = feed.getEntries();
System.out.println(feed.getEntries().size());
// select the particular spreadsheet on Inertia Ontology
for (SpreadsheetEntry spreadsheetEntry : spreadsheets) {
System.out.println("founded");
}
}
}
But when I run the code the size is 0 and no spreadsheet is printed out.
I have also make another test coding a program that access (on localhost) the Spreadsheet list using my Google Credential instead the OAuth2 authorization and it works well (it prints out all my spreadsheet on my Google Drive).
service.setUserCredentials(username, password); //My personal Google Account
NB: when I deploy on a remote server the test-program with My personal Google Credential (not OAuth2), it was blocked by Google for security policies.
Please any ideas?
Thank you

try to modify this snippet
GoogleCredential credential = new GoogleCredential.Builder()
.setTransport(transport)
.setJsonFactory(jsonFactory)
.setServiceAccountId("google-dev-console-email-address")
.setServiceAccountUser("your-gmail")
.setServiceAccountScopes(SCOPES)
.setServiceAccountPrivateKeyFromP12File(cert).build();

So when I deploy my code to CloudBees global.java won't run (Play Framework 2.1)

I'm just toying around with the google drive api (cloud storage) and figuring out how things work, I wrote some code in play framework where in global.java onStart access to the drive is gained and a test document is written to the drive. Locally this works fine but when I deploy my code to a new CloudBees app and visit the app I get this error in my log and I just can't figure out how to debug it:
Play server process ID is 7454
Oops, cannot start the server.
#6eonea90e: Cannot init the Global object
at play.api.WithDefaultGlobal$$anonfun$play$api$WithDefaultGlobal$$globalInstance$1.apply(Application.scala:57)
at play.api.WithDefaultGlobal$$anonfun$play$api$WithDefaultGlobal$$globalInstance$1.apply(Application.scala:51)
at play.utils.Threads$.withContextClassLoader(Threads.scala:18)
at play.api.WithDefaultGlobal$class.play$api$WithDefaultGlobal$$globalInstance(Application.scala:50)
at play.api.DefaultApplication.play$api$WithDefaultGlobal$$globalInstance$lzycompute(Application.scala:383)
at play.api.DefaultApplication.play$api$WithDefaultGlobal$$globalInstance(Application.scala:383)
at play.api.WithDefaultGlobal$class.global(Application.scala:66)
Link to complete CloudBees log
This is the code of Global.java:
import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.googleapis.auth.oauth2.GoogleTokenResponse;
import com.google.api.client.http.FileContent;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.drive.Drive;
import com.google.api.services.drive.DriveScopes;
import com.google.api.services.drive.model.File;
import com.google.api.services.drive.Drive.Files;
import com.google.api.services.drive.model.FileList;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.List;
import play.*;
import play.Logger;
public class Global extends GlobalSettings {
private static String CLIENT_ID = "595172328396-l4kpto8ip9fpaea0k2987eeq8f42bged.apps.googleusercontent.com";
private static String CLIENT_SECRET = "EvTUvAodjGx2eW_d3k8oy8Fb";
private static String REDIRECT_URI = "urn:ietf:wg:oauth:2.0:oob";
#Override
public void onStart(Application app) { try{
Logger.info("Application has started");
HttpTransport httpTransport = new NetHttpTransport();
JsonFactory jsonFactory = new JacksonFactory();
GoogleCredential credential = new GoogleCredential.Builder()
.setClientSecrets(CLIENT_ID, CLIENT_SECRET)
.setJsonFactory(jsonFactory).setTransport(httpTransport).build()
.setRefreshToken("1/MZ4GTNA_HMbOcKDSqp6ymSd11dlkgxoMXxfWwhwMJRg").setAccessToken("ya29.AHES6ZQk7NDC-OCba7_yANc_uqWPLwDLl95TlT_DXgkLqrr6qmyLRw");;
//Create a new authorized API client
Drive service = new Drive.Builder(httpTransport, jsonFactory, credential).build();
//Insert a file
File body = new File();
body.setTitle("New Document");
body.setDescription("A test document");
body.setMimeType("text/plain");
java.io.File fileContent = new java.io.File("document.txt");
FileContent mediaContent = new FileContent("text/plain", fileContent);
File file = service.files().insert(body, mediaContent).execute();
Logger.info("List of stuff: " + service.children().toString());
Logger.info("File ID: " + file.getId()); }catch (IOException ex) {}
}
}

The google access token is only good for 1 hour. It does not seem as if you are actually requesting an access token with your refresh token... Instead it seems like you are setting one that was already issued.
You need to request an access token be fetched (using the refresh token)
The "fun" is that once you use the refresh token to generate a new access token, the old ones are invalidated... Or at least that is the experience I have found...