Develop Reference

Where the Field "value" is declared?

I'm new on apache storm and kafka and try to learn these notions via courses provided by OpenClassroom.The principle is simple, messages are sent via a python program to a kafka server, and are retrieved via a kafka spout defined in the main class of a Storm topology. The problem is that I don't understand how the bolt retrieves the messages. From what I understand this is done in the ParsingBolt class with the following line of code: JSONObject obj = (JSONObject)jsonParser.parse(input.getStringByField("value"));. The only thing is that I don't understand how we know that the messages are contained in the value field. Below you can find the main class and the parsing bolt class. (The whole project is available here)
The main class:
package velos;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.AlreadyAliveException;
import org.apache.storm.generated.AuthorizationException;
import org.apache.storm.generated.InvalidTopologyException;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.kafka.spout.KafkaSpout;
import org.apache.storm.kafka.spout.KafkaSpoutConfig;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.topology.base.BaseWindowedBolt;
import org.apache.storm.tuple.Fields;
public class App {
public static void main(String[] args)
throws AlreadyAliveException, InvalidTopologyException, AuthorizationException {
TopologyBuilder builder = new TopologyBuilder();
KafkaSpoutConfig.Builder<String, String> spoutConfigBuilder = KafkaSpoutConfig.builder("localhost:9092",
"velib-stations");
spoutConfigBuilder.setProp(ConsumerConfig.GROUP_ID_CONFIG, "city-stats");
KafkaSpoutConfig<String, String> spoutConfig = spoutConfigBuilder.build();
builder.setSpout("stations", new KafkaSpout<String, String>(spoutConfig));
builder.setBolt("station-parsing", new StationParsingBolt()).shuffleGrouping("stations");
builder.setBolt("city-stats",
new CityStatsBolt().withTumblingWindow(BaseWindowedBolt.Duration.of(1000 * 60 * 5)))
.fieldsGrouping("station-parsing", new Fields("city"));
builder.setBolt("save-results", new SaveResultsBolt()).fieldsGrouping("city-stats", new Fields("city"));
StormTopology topology = builder.createTopology();
Config config = new Config();
config.setMessageTimeoutSecs(60 * 30);
String topologyName = "Velos";
if (args.length > 0 && args[0].equals("remote")) {
StormSubmitter.submitTopology(topologyName, config, topology);
} else {
LocalCluster cluster = new LocalCluster();
cluster.submitTopology(topologyName, config, topology);
}
}
}
The ParsingBolt:
package velos;
import java.util.Map;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
import org.apache.storm.shade.org.json.simple.JSONObject;
import org.apache.storm.shade.org.json.simple.parser.JSONParser;
import org.apache.storm.shade.org.json.simple.parser.ParseException;
public class StationParsingBolt extends BaseRichBolt {
private OutputCollector outputCollector;
#Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
outputCollector = collector;
}
#Override
public void execute(Tuple input) {
try {
process(input);
} catch (ParseException e) {
e.printStackTrace();
outputCollector.fail(input);
}
}
public void process(Tuple input) throws ParseException {
JSONParser jsonParser = new JSONParser();
JSONObject obj = (JSONObject)jsonParser.parse(input.getStringByField("value"));
String contract = (String)obj.get("contract_name");
Long availableStands = (Long)obj.get("available_bike_stands");
Long stationNumber = (Long)obj.get("number");
outputCollector.emit(new Values(contract, stationNumber, availableStands));
outputCollector.ack(input);
}
#Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("city", "station_id", "available_stands"));
}
}

By default the "topic", "partition", "offset", "key", and "value" will be emitted to the "default" stream.
https://storm.apache.org/releases/2.4.0/storm-kafka-client.html
Use a RecordTranslator to change this.

API of Kubernetes in java: authentication error

I am developing a simple java app that show the pods of the cluster.
This is the app:
import io.kubernetes.client.openapi.ApiClient;
import io.kubernetes.client.openapi.ApiException;
import io.kubernetes.client.openapi.Configuration;
import io.kubernetes.client.openapi.apis.CoreV1Api;
import io.kubernetes.client.openapi.models.V1Pod;
import io.kubernetes.client.openapi.models.V1PodList;
import io.kubernetes.client.util.ClientBuilder;
import io.kubernetes.client.util.KubeConfig;
import java.io.FileReader;
import java.io.IOException;
/**
* A simple example of how to use the Java API from an application outside a kubernetes cluster
*
* <p>Easiest way to run this: mvn exec:java
* -Dexec.mainClass="io.kubernetes.client.examples.KubeConfigFileClientExample"
*
*/
public class untitled4 {
public static void main(String[] args) throws IOException, ApiException {
// file path to your KubeConfig
String kubeConfigPath = "/home/robin/.kube/config";
// loading the out-of-cluster config, a kubeconfig from file-system
ApiClient client =
ClientBuilder.kubeconfig(KubeConfig.loadKubeConfig(new FileReader(kubeConfigPath))).build();
// set the global default api-client to the in-cluster one from above
Configuration.setDefaultApiClient(client);
// the CoreV1Api loads default api-client from global configuration.
CoreV1Api api = new CoreV1Api();
// invokes the CoreV1Api client
V1PodList list = api.listPodForAllNamespaces(null, null, null, null, null, null, null, null, null);
System.out.println("Listing all pods: ");
for (V1Pod item : list.getItems()) {
System.out.println(item.getMetadata().getName());
}
}
}
But I get this error:
Exception in thread "main" java.lang.IllegalStateException: Unimplemented
at io.kubernetes.client.util.authenticators.GCPAuthenticator.refresh(GCPAuthenticator.java:61)
at io.kubernetes.client.util.KubeConfig.getAccessToken(KubeConfig.java:215)
at io.kubernetes.client.util.credentials.KubeconfigAuthentication.<init>(KubeconfigAuthentication.java:46)
at io.kubernetes.client.util.ClientBuilder.kubeconfig(ClientBuilder.java:276)
at untitled4.main(untitled4.java:28)
Process finished with exit code 1

There is an open issue on GitHub related with this problem. For now you can use workarounds like the one, proposed by jhbae200 in this comment:
I am using it like this.
package kubernetes.gcp;
import com.google.auth.oauth2.AccessToken;
import com.google.auth.oauth2.GoogleCredentials;
import io.kubernetes.client.util.KubeConfig;
import io.kubernetes.client.util.authenticators.Authenticator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.time.Instant;
import java.util.Date;
import java.util.Map;
public class ReplacedGCPAuthenticator implements Authenticator {
private static final Logger log;
private static final String ACCESS_TOKEN = "access-token";
private static final String EXPIRY = "expiry";
static {
log = LoggerFactory.getLogger(io.kubernetes.client.util.authenticators.GCPAuthenticator.class);
}
private final GoogleCredentials credentials;
public ReplacedGCPAuthenticator(GoogleCredentials credentials) {
this.credentials = credentials;
}
public String getName() {
return "gcp";
}
public String getToken(Map<String, Object> config) {
return (String) config.get("access-token");
}
public boolean isExpired(Map<String, Object> config) {
Object expiryObj = config.get("expiry");
Instant expiry = null;
if (expiryObj instanceof Date) {
expiry = ((Date) expiryObj).toInstant();
} else if (expiryObj instanceof Instant) {
expiry = (Instant) expiryObj;
} else {
if (!(expiryObj instanceof String)) {
throw new RuntimeException("Unexpected object type: " + expiryObj.getClass());
}
expiry = Instant.parse((String) expiryObj);
}
return expiry != null && expiry.compareTo(Instant.now()) <= 0;
}
public Map<String, Object> refresh(Map<String, Object> config) {
try {
AccessToken accessToken = this.credentials.refreshAccessToken();
config.put(ACCESS_TOKEN, accessToken.getTokenValue());
config.put(EXPIRY, accessToken.getExpirationTime());
} catch (IOException e) {
throw new RuntimeException(e);
}
return config;
}
}
Running in.
//GoogleCredentials.fromStream(--something credential.json filestream--)
KubeConfig.registerAuthenticator(new ReplacedGCPAuthenticator(GoogleCredentials.getApplicationDefault()));
ApiClient client = Config.defaultClient();
Configuration.setDefaultApiClient(client);
CoreV1Api api = new CoreV1Api();
V1PodList list = api.listNamespacedPod("default", null, null, null, null, null, null, null, 30, Boolean.FALSE);
for (V1Pod item : list.getItems()) {
System.out.println(item.getMetadata().getName());
}

HttpServer with more than one responder

In my application (java) I need to support two types of RPCs:
Administration RPC and User RPC. I am using avro to create these RPCs, currently I am opening two HttpServers one per RPC using code similar to the following:
Server serverAdmins = new HttpServer(new ReflectResponder(AdministrationRPC.class, arpcImpl), adminRpcPort);
Server serverUsers = new HttpServer(new ReflectResponder(UsersRPC.class, urpcImpl), usersRpcPort);
...
This works, but it look like a waste for me - I am looking for a way to use a single http server with two ports or two urls on the same port (any of these options is good for me), something like:
Server server = new new HttpServer(new ReflectResponder(AdministrationRPC.class, arpcImpl), adminRpcPort);
server.addResponder(new ReflectResponder(UsersRPC.class, urpcImpl), usersRpcPort);
...
the addResponder method does not exists of course, the only method that looks similar is the addConnector method - but even after thorough googling I couldn't find how to apply it to my needs..
Is there a way to start two avro responders on the same http server?

I managed to resolve this issue by writing a simple class which allow for what I want, see code below.
import java.io.IOException;
import java.net.URL;
import org.apache.avro.ipc.HttpTransceiver;
import org.apache.avro.ipc.Responder;
import org.apache.avro.ipc.ResponderServlet;
import org.apache.avro.ipc.reflect.ReflectRequestor;
import org.apache.avro.ipc.reflect.ReflectResponder;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.servlet.Context;
import org.mortbay.jetty.servlet.ServletHolder;
/**
*
* #author bennyl
*/
public class MultiResponderHttpServer {
private final Context context;
private final Server server;
private final int port;
public MultiResponderHttpServer(int port) {
this.port = port;
server = new Server(port);
context = new Context(server, "/", Context.SESSIONS);
}
public void addResponder(String baseUri, Responder responder) throws IOException {
ResponderServlet servlet = new ResponderServlet(responder);
ServletHolder holder = new ServletHolder(servlet);
context.addServlet(holder, baseUri);
}
public int getPort() {
return port;
}
public void close() throws Exception {
server.stop();
}
public void start() throws Exception {
server.start();
}
public void join() throws InterruptedException {
server.join();
}
public static void main(String[] args) throws IOException, InterruptedException {
MultiResponderHttpServer server = new MultiResponderHttpServer(8888);
server.addResponder("/test_a/*", new ReflectResponder(TestProtocol.class,
(TestProtocol) why -> "a received a message: '" + why + "'"));
server.addResponder("/test_b/*", new ReflectResponder(TestProtocol.class,
(TestProtocol) why -> "b received a message: '" + why + "'"));
server.start();
HttpTransceiver atrans = new HttpTransceiver(new URL("http://localhost:" + server.getPort() + "/test_a/"));
HttpTransceiver btrans = new HttpTransceiver(new URL("http://localhost:" + server.getPort() + "/test_b/"));
System.out.println(ReflectRequestor.getClient(TestProtocol.class, atrans).go("message to a"));
System.out.println(ReflectRequestor.getClient(TestProtocol.class, btrans).go("message to b"));
server.close();
server.join();
}
public interface TestProtocol {
String go(String why);
}
}

Error while attempting to program page creation on google sites

I'm trying to programmatic-ally add pages to my Google Site using Java.
This is the code:
import java.io.*;
import java.net.MalformedURLException;
import java.net.URL;
import com.google.gdata.client.sites.*;
import com.google.gdata.data.PlainTextConstruct;
import com.google.gdata.data.XhtmlTextConstruct;
import com.google.gdata.data.sites.*;
import com.google.gdata.util.ServiceException;
import com.google.gdata.util.XmlBlob;
public class PageCreate {
public static void main(String args[]) throws Exception {
WebPageEntry createdEntry = createWebPage("New Webpage Title", "<b>HTML content</b>");
System.out.println("Created! View at " + createdEntry.getHtmlLink().getHref());
}
private static void setContentBlob(BaseContentEntry<?> entry, String pageContent) {
XmlBlob xml = new XmlBlob();
xml.setBlob(pageContent);
entry.setContent(new XhtmlTextConstruct());
}
public static WebPageEntry createWebPage(String title, String content)
throws ServiceException, IOException, MalformedURLException {
SitesService client = new SitesService("*****-pagecreate-v1");
client.setUserCredentials("***********", "*********");
client.site = "intratrial2"; -> ***SYNTAX ERROR REPORTED***
//ContentFeed contentFeed = client.getFeed(new URL(buildContentFeedUrl()), ContentFeed.class);
WebPageEntry entry = new WebPageEntry();
entry.setTitle(new PlainTextConstruct(title));
setContentBlob(entry, content); // Entry's HTML content
return client.insert(new URL(buildContentFeedUrl()), entry);
}
public static String buildContentFeedUrl() {
String domain = "*****"; // OR if the Site is hosted on Google Apps, your domain (e.g. example.com)
String siteName = "intratrial2";
return "https://sites.google.com/feeds/content/" + domain + "/" + siteName + "/";
}
}
If i comment out the line with syntax error i get the following error report on running:
Exception in thread "main" com.google.gdata.util.ServiceException: Internal Server Error
Internal Error
I'm not sure what I'm doing wrong here and I'd really appreciate some help. Thanks.

Magento XML-RPC API - Create Shipment From Java

Im trying to create shipments for magento orders from my third party app using the XML-RPC API. Everything works great when i make the call to "sales_order_shipment.create" with just the order increment id, but if i try making the same call with both an order increment id and a set of items and quantities, it will say "Requested order not exists." Why is that? what im doing wrong? what type should be the itemQuantity)
heres my code
package magentoapiclient;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.xmlrpc.XmlRpcException;
import org.apache.xmlrpc.client.XmlRpcClient;
import org.apache.xmlrpc.client.XmlRpcClientConfigImpl;
public class XMLRPCAPIClient {
public static void main(String[] args) {
createShipment("100000005", 5, 1.0);
}
public static XmlRpcClient prepareClient() throws MalformedURLException {
XmlRpcClientConfigImpl config = new XmlRpcClientConfigImpl();
config.setServerURL(new URL("myHost"));
XmlRpcClient client = new XmlRpcClient();
client.setConfig(config);
client.setTypeFactory(new MyTypeFactory(client));
return client;
}
public static void createShipment(String orderIncrementId, int itemId, double quantity) {
try {
XmlRpcClient client = prepareClient();
String sessionId = login("myUser", "myKey", client);
System.out.println(sessionId);
Object[] request = {orderIncrementId, new HashMap()};
client.execute("call", new Object[]{sessionId, "sales_order_shipment.create", request});
endSession(sessionId, client);
} catch (XmlRpcException | MalformedURLException ex) {
Logger.getLogger(XMLRPCAPIClient.class.getName()).log(Level.SEVERE, null, ex);
}
}
public static String login(String user, String password, XmlRpcClient client) throws XmlRpcException {
String sessionId = (String) client.execute("login", new Object[]{user, password});
return sessionId;
}
public static void endSession(String sessionToken, XmlRpcClient client) throws XmlRpcException {
client.execute("endSession", new Object[]{sessionToken});
}
}
As you see in the line
Object[] request = {orderIncrementId, new HashMap()};
i try sending a hashMap that actually should contain the order item id and quantity. Ive also tried sending an array of objects and it doesnt work for any of them. What sould be there instead then if neither a Map or an Array is available?
Thank you for your help

Order Id and Order increment id are different. You should send order increment id.
http://www.magentocommerce.com/api/soap/sales/salesOrder/sales_order.info.html