I'm trying to set up a server on aws with simple http server and save each http post request headers & payload.
It works locally.
My steps after connection via ssh to the ec2 server:
javac Server.java
sudo nohup java Server
It saves the headers to log file but not the payload and it doesn't returns 204 response.
Server.java
import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpHandler;
import com.sun.net.httpserver.HttpServer;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.net.URLDecoder;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.Executors;
public class Server {
private static final int PORT = 80;
private static final String FILE_PATH = "/home/ec2-user/logs/";
private static final String UTF8 = "UTF-8";
private static final String DELIMITER = "|||";
private static final String LINE_BREAK = "\n";
private static final String FILE_PREFIX = "dd_MM_YYYY_HH";
private static final SimpleDateFormat simpleDateFormat = new SimpleDateFormat(FILE_PREFIX);
private static final String FILE_TYPE = ".txt";
public static void main(String[] args) {
try {
HttpServer server = HttpServer.create(new InetSocketAddress(PORT), 0);
server.createContext("/", new HttpHandler() {
#Override
public void handle(HttpExchange t) throws IOException {
System.out.println("Req\t" + t.getRemoteAddress());
InputStream initialStream = t.getRequestBody();
byte[] buffer = new byte[initialStream.available()];
initialStream.read(buffer);
File targetFile = new File(FILE_PATH + simpleDateFormat.format(new Date()) + FILE_TYPE);
OutputStream outStream = new FileOutputStream(targetFile, true);
String prefix = LINE_BREAK + t.getRequestHeaders().entrySet().toString() + LINE_BREAK + System.currentTimeMillis() + DELIMITER;
outStream.write(prefix.getBytes());
Map<String, String> queryPairs = new HashMap<>();
String params = new String(buffer);
String[] pairs = params.split("&");
for (String pair : pairs) {
int idx = pair.indexOf("=");
String key = pair.substring(0, idx);
String val = pair.substring(idx + 1);
String decodedKey = URLDecoder.decode(key, UTF8);
String decodeVal = URLDecoder.decode(val, UTF8);
queryPairs.put(decodedKey, decodeVal);
}
outStream.write(queryPairs.toString().getBytes());
t.sendResponseHeaders(204, -1);
t.close();
}
});
server.setExecutor(Executors.newCachedThreadPool());
server.start();
} catch (Exception e) {
System.out.println("Exception: " + e.getMessage());
e.printStackTrace();
}
}
}
Consider these changes to your handle method. As a starting point, two things are changed:
It reads the complete input and copies that into your file (initialStream.available() might not be the full truth)
catch, log and rethrow IOExceptions (you didn't see your 204 after all)
Consider redirecting your output into files, so you can check what happend on server later:
sudo nohup java Server > server.log 2> server.err &
If you described in more detail the desired target file structure we could figure something out there as well I guess.
#Override
public void handle(HttpExchange t) throws IOException {
try {
System.out.println("Req\t" + t.getRemoteAddress());
InputStream initialStream = t.getRequestBody();
File targetFile = new File(FILE_PATH + simpleDateFormat.format(new Date()) + FILE_TYPE);
OutputStream outStream = new FileOutputStream(targetFile, true);
// This will copy ENTIRE input stream into your target file
IOUtils.copy(initialStream, outStream);
outStream.close();
t.sendResponseHeaders(204, -1);
t.close();
} catch(IOException e) {
e.printStackTrace();
throw e;
}
}
Related
So I have a client written in Java that i want to use to test out sending email but instead of using an already existing SMTP like google, i want to have my own local server to test out sending mock emails between two mock emails.
I've been trying to look all over the internet for good sources on how to code a simple SMTP Server but i've had zero luck.
I do have a basic server code that when i run it, i can connect my Client to it but at the moment it won't handle any email functionality.
TCPServer.java
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.InetAddress;
import java.net.ServerSocket;
import java.net.Socket;
import java.io.*;
import java.net.*;
public class TCPServer{
private ServerSocket server;
/**
* The TCPServer constructor initiate the socket
* #param ipAddress
* #param port
* #throws Exception
*/
public TCPServer(String ipAddress, int port) throws Exception {
if (ipAddress != null && !ipAddress.isEmpty())
this.server = new ServerSocket(port, 1, InetAddress.getByName(ipAddress));
else
this.server = new ServerSocket(0, 1, InetAddress.getLocalHost());
}
/**
* The listen method listen to incoming client's datagrams and requests
* #throws Exception
*/
private void listen() throws Exception {
// listen to incoming client's requests via the ServerSocket
//add your code here
String data = null;
Socket client = this.server.accept();
String clientAddress = client.getInetAddress().getHostAddress();
System.out.println("\r\nNew client connection from " + clientAddress);
// print received datagrams from client
//add your code here
BufferedReader in = new BufferedReader(new InputStreamReader(client.getInputStream()));
while ( (data = in.readLine()) != null ) {
System.out.println("\r\nMessage from " + clientAddress + ": " + data);
client.sendUrgentData(1);
}
}
public InetAddress getSocketAddress() {
return this.server.getInetAddress();
}
public int getPort() {
return this.server.getLocalPort();
}
public static void main(String[] args) throws Exception {
// set the server address (IP) and port number
//add your code here
String serverIP = "192.168.1.235"; // local IP address
int port = 8088;
if (args.length > 0) {
serverIP = args[0];
port = Integer.parseInt(args[1]);
}
// call the constructor and pass the IP and port
//add your code here
TCPServer server = new TCPServer(serverIP, port);
System.out.println("\r\nRunning Server: " +
"Host=" + server.getSocketAddress().getHostAddress() +
" Port=" + server.getPort());
server.listen();
}
}
What can i add to my existing server code to make it handle email for my Client. I'll also post my email client as well.
ClientTester.java
import java.io.*;
import java.net.DatagramSocket;
import java.net.InetAddress;
import java.net.Socket;
import java.util.Scanner;
/**
* This program demonstrates a TCP client
* #author jl922223
* #version 1.0
* #since 2020-12-12
*/
public class ClientTester{
private Socket tcpSocket;
private InetAddress serverAddress;
private int serverPort;
private Scanner scanner;
/**
* #param serverAddress
* #param serverPort
* #throws Exception
*/
private ClientTester(InetAddress serverAddress, int serverPort) throws Exception {
this.serverAddress = serverAddress;
this.serverPort = serverPort;
//Initiate the connection with the server using Socket.
//For this, creates a stream socket and connects it to the specified port number at the specified IP address.
//add your code here
this.tcpSocket = new Socket(this.serverAddress, this.serverPort);
this.scanner = new Scanner(System.in);
}
/**
* The start method connect to the server and datagrams
* #throws IOException
*/
/* private void start() throws IOException {
String input;
//create a new PrintWriter from an existing OutputStream (i.e., tcpSocket).
//This convenience constructor creates the necessary intermediateOutputStreamWriter, which will convert characters into bytes using the default character encoding
//You may add your code in a loop so that client can keep send datagrams to server
//add your code here
while (true) {
System.out.print ("C:");
input = scanner.nextLine();
PrintWriter output = new PrintWriter(this.tcpSocket.getOutputStream(), true);
output.println(input);
output.flush();
}
}*/
public static void main(String[] args) throws Exception {
// set the server address (IP) and port number
//add your code here
//IP: 192.168.1.235
//Port: 8088
InetAddress serverIP = InetAddress.getByName("smtp.google.com"); // local IP address
int port = 25;
if (args.length > 0) {
serverIP = InetAddress.getByName(args[0]);
port = Integer.parseInt(args[1]);
}
// call the constructor and pass the IP and port
//add your code here
ClientTester client = new ClientTester(serverIP, port);
// client.start();
try{
client = new ClientTester(serverIP, port);
System.out.println("\r\n Connected to Server: " + client.tcpSocket.getInetAddress());
BufferedReader stdin;
stdin = new BufferedReader (new InputStreamReader (System.in));
InputStream is = client.tcpSocket.getInputStream ();
BufferedReader sockin;
sockin = new BufferedReader (new InputStreamReader (is));
OutputStream os = client.tcpSocket.getOutputStream();
PrintWriter sockout;
sockout = new PrintWriter (os, true);
System.out.println ("S:" + sockin.readLine ());
while (true){
System.out.print ("C:");
String cmd = stdin.readLine ();
sockout.println (cmd);
String reply = sockin.readLine ();
System.out.println ("S:" + reply);
if (cmd.toLowerCase ().startsWith ("data") &&
reply.substring (0, 3).equals ("354"))
{
do
{
cmd = stdin.readLine ();
if (cmd != null && cmd.length () > 1 &&
cmd.charAt (0) == '.')
cmd = "."; // Must be no chars after . char.
sockout.println (cmd);
if (cmd.equals ("."))
break;
}
while (true);
// Read a reply string from the SMTP server program.
reply = sockin.readLine ();
// Display the first line of this reply string.
System.out.println ("S:" + reply);
continue;
}
// If the QUIT command was entered, quit.
if (cmd.toLowerCase ().startsWith ("quit"))
break;
}
}
catch (IOException e)
{
System.out.println (e.toString ());
}
finally
{
try
{
// Attempt to close the client socket.
if (client != null)
client.tcpSocket.close();
}
catch (IOException e)
{
}
}
}
}
The good news is that the ClientTester works when i connect it to smtp.google.com but i don't want to use Googles, i want to have my own basic Email server in java.
Okay, found this early-development standalone version.
Use this INSTEAD of your code; does everything your code does an more.
Single-threaded ServerSocket handling, so only one connection at a time.
package jc.lib.io.net.email.smtp.test1;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.net.ServerSocket;
import java.net.Socket;
import java.net.SocketTimeoutException;
import java.net.UnknownHostException;
import java.text.SimpleDateFormat;
import java.util.Date;
import jc.lib.io.net.email.JcEMailBasics;
public class Test_SMTP_Server {
static public boolean DEBUG = true;
public static void main(final String s[]) throws UnknownHostException, IOException {
final Test_SMTP_Server server = new Test_SMTP_Server(JcEMailBasics.SMTP_PORTS);
server.start();
try {
Thread.sleep(1 * 60 * 60 * 1000);
} catch (final InterruptedException e) { /* */ }
}
/*
* OBJECT
*/
private final ServerSocket[] mSockets;
private volatile boolean mStopRequested;
private static boolean mReceivingData;
public Test_SMTP_Server(final int[] pPorts) throws IOException {
mSockets = new ServerSocket[pPorts.length];
for (int i = 0; i < pPorts.length; i++) {
final int port = pPorts[i];
try {
mSockets[i] = new ServerSocket(port);
} catch (final java.net.BindException e) {
new java.net.BindException("When mountin port " + port + ": " + e.getMessage()).printStackTrace();
}
System.out.println("Created server socket on port " + port);
}
}
public void start() {
mStopRequested = false;
for (final ServerSocket ss : mSockets) {
if (ss == null) continue;
final Thread t = new Thread(() -> handleServerSocket(ss), "handleServerSocket(" + ss.getLocalPort() + ")");
t.setDaemon(true);
t.start();
}
}
private void handleServerSocket(final ServerSocket pSS) {
final String name = "handleServerSocket(" + pSS.getLocalPort() + ")";
while (!mStopRequested) {
System.out.println(name + "\tListening for connection...");
try (final Socket socket = pSS.accept();
final BufferedReader in = new BufferedReader(new InputStreamReader(socket.getInputStream(), JcEMailBasics.DEFAULT_CHARSET_SMTP_POP3));
final BufferedWriter out = new BufferedWriter(new OutputStreamWriter(socket.getOutputStream(), JcEMailBasics.DEFAULT_CHARSET_SMTP_POP3));) {
System.out.println(name + "\tGot new Socket.");
handle(socket, in, out);
System.out.println(name + "\tClosing Socket.");
} catch (final IOException e) {
System.err.println("In " + name + ":");
e.printStackTrace();
}
System.out.println(name + "\tComm Done.");
}
}
public void stop() {
mStopRequested = true;
for (final ServerSocket ss : mSockets) {
try {
ss.close();
} catch (final Exception e) { /* */ }
}
}
static private void handle(final Socket pSocket, final BufferedReader pBR, final BufferedWriter pBW) throws IOException {
// send("+OK POP3 server ready <" + Test_EMails.SERVICE_ADDRESS + ">", out);
send("220 cbsoft.dev SMTP " + JcEMailBasics.NAME, pBW);
final StringBuilder sb = new StringBuilder();
mainLoop: while (!pSocket.isClosed()) {
final String read = read(pBR);
if (read == null) break;
switch (read) {
case JcEMailBasics.COMMAND_DATA: {
send("354 End data with <CR><LF>.<CR><LF>", pBW);
mReceivingData = true;
break;
}
case JcEMailBasics.COMMAND_END_OF_DATA: {
send("250 OK", pBW);
mReceivingData = false;
break;
}
case JcEMailBasics.COMMAND_QUIT: {
send("221 " + JcEMailBasics.NAME + " signing off", pBW);
break mainLoop;
}
default: {
final String correctedRead = read.startsWith(".") ? read.substring(1) : read;
sb.append(correctedRead + "\n");
if (!mReceivingData) send("250 Ok", pBW);
}
}
}
final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss");
final File file = new File("mails/inc_" + sdf.format(new Date()) + ".email.txt");
file.getParentFile().mkdirs();
final String msg = sb.toString();
try (FileOutputStream fos = new FileOutputStream(file)) {
fos.write(msg.getBytes());
}
System.out.println("File saved as " + file.getCanonicalPath());
}
static private void send(final String pMessage, final BufferedWriter pBW) {
try {
pBW.write(pMessage + "\n");
pBW.flush();
if (DEBUG) System.out.println("SENT:\t" + pMessage);
} catch (final Exception e) {
e.printStackTrace();
}
}
static private String read(final BufferedReader pBR) throws IOException {
try {
final String reply = pBR.readLine();
if (DEBUG) System.out.println("RECV:\t" + reply);
return reply;
} catch (final SocketTimeoutException e) {
System.err.println("SERVER TIMEOUT");
}
return null;
}
}
the only additional file you will need (also included in my previous answer; edited a bit):
package jc.lib.io.net.email;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.IOException;
public class JcEMailBasics {
static public final int SMTP_PORT_1 = 25;
static public final int SMTP_PORT_2 = 587;
static public final int SMTP_PORT_3 = 465;
static public final int[] SMTP_PORTS = { SMTP_PORT_1, SMTP_PORT_2, SMTP_PORT_3 };
static public final int POP_PORT_1 = 110;
static public final int POP_PORT_SSL = 995;
static public final int POP_PORT_KERBEROS = 1109;
static public final int[] POP_PORTS = { POP_PORT_1, POP_PORT_SSL, POP_PORT_KERBEROS };
// netstat -aon | findstr '587'
static public final String DEFAULT_CHARSET_SMTP_POP3 = "8859_1";
static public final String NAME = "JC Oblivionat0r POP3 Server";
static public final String SERVICE_ADDRESS = "oblivionat0r#cbsoft.dev";
static public final String CONNECTION_CLOSED = "CONNECTION_CLOSED_dtnt495n3479r5zb3tr47c3b49c3";
static public final String COMMAND_QUIT = "QUIT";
static public final String COMMAND_DATA = "DATA";
static public final String COMMAND_END_OF_DATA = ".";
static public void send(final BufferedWriter pBufferedWriter, final String pMessage) throws IOException {
pBufferedWriter.write(pMessage + "\n");
pBufferedWriter.flush();
System.out.println("SENT:\t" + pMessage);
}
static public String sendExpect(final BufferedWriter pBufferedWriter, final String pMessage, final BufferedReader pBufferedReader, final String... pExpectedResponsePrefixes) throws IOException {
send(pBufferedWriter, pMessage);
final String read = read(pBufferedReader);
for (final String erp : pExpectedResponsePrefixes) {
if (read.startsWith(erp)) return read;
}
throw new IllegalStateException("Bad response: Expected [" + toString(", ", pExpectedResponsePrefixes) + "] got [" + read + "] instead!");
}
static public String read(final BufferedReader pBufferedReader) throws IOException {
final String reply = pBufferedReader.readLine();
System.out.println("RECV:\t" + reply);
return reply;
}
#SafeVarargs public static <T> String toString(final String pSeparator, final T... pObjects) {
if (pObjects == null) return null;
final StringBuilder ret = new StringBuilder();
for (final T o : pObjects) {
ret.append(o + pSeparator);
}
if (ret.length() > 0) ret.setLength(ret.length() - pSeparator.length());
return ret.toString();
}
}
Basically like my code.
It is just a proof of concept, and quite unsafe and inefficient
I'm using lombok. The read() method is basically a BufferedReader.readLine() call on the socket's InputStream.
send() is a writeLine
My entry point handleSocket() is when the Socket connection is established.
The String.toNLine() method is a Lombok extension, you can replace it with string.replace("\r\n" , "\n");
Be aware that this is simply a stupid implementation that can be fooled easily, but it enables basic email receiving. You get ALL the communication in the StringBuilder. You could take that final whole text apart with MIME classes (Header / newline / newline body method that is used by HTTP, SMTP etc).
This approach collects the whole comunication first, then later (outside given code) handles the actual MIME part. You could also implement it differently, as in the code knows the current state of transmission and details of the MIME object it's currently receiving, and updates its status/workflow with each line. That would be much more efficient, but the code would be a bit more complex.
package jc.lib.io.net.email.smtp.server.receiver;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.ServerSocket;
import java.net.Socket;
import java.text.SimpleDateFormat;
import java.util.Date;
import jc.lib.aop.lombok.java.lang.JcAString;
import jc.lib.collection.tuples.JcTriple;
import jc.lib.io.net.email.JcEMailBasics;
import jc.lib.io.net.email.util.JcAServerSocketHandlerBase;
import jc.lib.lang.thread.event.JcEvent;
import lombok.experimental.ExtensionMethod;
#ExtensionMethod({ JcAString.class })
public class JcSmtpReceiverSocketHandler extends JcAServerSocketHandlerBase {
public final JcEvent<JcTriple<JcSmtpReceiver, JcSmtpReceiverSocketHandler, File>> EVENT_EMAIL_RECEIVED = new JcEvent<>();
private final JcSmtpReceiver mJcAServerBase;
private boolean mReceivingData;
public JcSmtpReceiverSocketHandler(final JcSmtpReceiver pJcAServerBase, final ServerSocket pServerSocket, final Socket pSocket) throws IOException {
super(pServerSocket, pSocket);
mJcAServerBase = pJcAServerBase;
}
#Override protected void handleSocket() throws IOException {
send("220 cbsoft.dev SMTP " + JcEMailBasics.NAME);
final StringBuilder sb = new StringBuilder();
mainLoop: while (!mSocket.isClosed()) {
final String read = read();
if (read == null) break;
switch (read) {
case JcEMailBasics.COMMAND_DATA: {
send("354 End data with <CR><LF>.<CR><LF>");
mReceivingData = true;
break;
}
case JcEMailBasics.COMMAND_END_OF_DATA: {
send("250 OK");
mReceivingData = false;
break;
}
case JcEMailBasics.COMMAND_QUIT: {
send("221 " + JcEMailBasics.NAME + " signing off");
break mainLoop;
}
default: {
final String correctedRead = read.startsWith(".") ? read.substring(1) : read;
sb.append(correctedRead + "\n");
if (!mReceivingData) send("250 Ok");
}
}
}
final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss");
final File file = new File("mails/inc_" + sdf.format(new Date()) + ".email.txt");
file.getParentFile().mkdirs();
String msg = sb.toString();
msg = msg.toNLineBreak();
final String header = msg.subStringBefore("\n\n");
System.out.println("header:");
try (FileOutputStream fos = new FileOutputStream(file)) {
fos.write(msg.getBytes());
}
System.out.println("File saved as " + file.getCanonicalPath());
EVENT_EMAIL_RECEIVED.trigger(new JcTriple<>(mJcAServerBase, this, file));
}
}
Check out this file for some ports and other info.
package jc.lib.io.net.email;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.IOException;
import jc.lib.io.net.email.util.JcAServerBase;
import jc.lib.lang.JcUArray;
public class JcEMailBasics {
static public final int SMTP_PORT_1 = 25;
static public final int SMTP_PORT_2 = 587;
static public final int SMTP_PORT_3 = 465;
static public final int[] SMTP_PORTS = { SMTP_PORT_1, SMTP_PORT_2, SMTP_PORT_3 };
static public final int POP_PORT_1 = 110;
static public final int POP_PORT_SSL = 995;
static public final int POP_PORT_KERBEROS = 1109;
static public final int[] POP_PORTS = { POP_PORT_1, POP_PORT_SSL, POP_PORT_KERBEROS };
// netstat -aon | findstr '587'
static public final String DEFAULT_CHARSET_SMTP_POP3 = "8859_1";
static public final String NAME = "JC Oblivionat0r POP3 Server";
static public final String SERVICE_ADDRESS = "oblivionat0r#cbsoft.dev";
static public final String CONNECTION_CLOSED = "CONNECTION_CLOSED_dtnt495n3479r5zb3tr47c3b49c3";
static public final String COMMAND_QUIT = "QUIT";
static public final String COMMAND_DATA = "DATA";
static public final String COMMAND_END_OF_DATA = ".";
static public void send(final BufferedWriter pBufferedWriter, final String pMessage) throws IOException {
pBufferedWriter.write(pMessage + "\n");
pBufferedWriter.flush();
if (JcAServerBase.DEBUG) System.out.println("SENT:\t" + pMessage);
}
static public String sendExpect(final BufferedWriter pBufferedWriter, final String pMessage, final BufferedReader pBufferedReader, final String... pExpectedResponsePrefixes) throws IOException {
send(pBufferedWriter, pMessage);
final String read = read(pBufferedReader);
for (final String erp : pExpectedResponsePrefixes) {
if (read.startsWith(erp)) return read;
}
throw new IllegalStateException("Bad response: Expected [" + JcUArray.toString(", ", pExpectedResponsePrefixes) + "] got [" + read + "] instead!");
}
static public String read(final BufferedReader pBufferedReader) throws IOException {
final String reply = pBufferedReader.readLine();
if (JcAServerBase.DEBUG) System.out.println("RECV:\t" + reply);
return reply;
}
}
You need to communicate with the client.
First let the server send something like "220 Smtp server" (only 220 matters) to the client.
I used PrintWriter:
PrintWriter out = new PrintWriter(client.getOutputStream(), true);
out.println("220 Smtp server");
Then you will receive an EHLO from the client while getting lines from the inputStream.
Here you can find an example of the communication between server an client (without the starting message from the server (220)):
https://postmarkapp.com/guides/everything-you-need-to-know-about-smtp#basic-smtp-commands
Could somebody point me, using Twilio Java API, NOT! REST requests, how can I get the recorded file (.wav) of a concrete call.
I have read all the related articles to recording (https://support.twilio.com/hc/en-us/sections/205104748-Recording), but none of them shows how to do that with Java API.
I use this code, as an starting point, assuming the CALL_SID is known:
import com.twilio.Twilio;
import com.twilio.base.ResourceSet;
import com.twilio.rest.api.v2010.account.Recording;
import com.twilio.rest.api.v2010.account.RecordingReader;
public class DeleteRecordings1 {
private static final String ACCOUNT_SID = "ACXXXXXXXXXXXXXXXXX";
private static final String AUTH_TOKEN = "999aa999aaa999aaaa999";
private static final String CALL_SID = "CA83837718818gdgdg";
public static void main(String[] args) {
try {
Twilio.init(ACCOUNT_SID, AUTH_TOKEN);
RecordingReader recordingReader = Recording.reader();
recordingReader.setCallSid(CALL_SID);
ResourceSet<Recording> recordings = recordingReader.read();
String recordingSid;
for (Recording recording: recordings) {
recordingSid = recording.getSid();
//HERE! I want to restore the .wav file associated with that RECORD_SID ?¿
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
I put the final code, in case it can help somebody:
import java.io.File;
import java.io.InputStream;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import com.twilio.Twilio;
import com.twilio.base.ResourceSet;
import com.twilio.rest.api.v2010.account.Recording;
import com.twilio.rest.api.v2010.account.RecordingReader;
public class GetCallRecordings {
private static final String ACCOUNT_SID = "ACXXXXXXXXXXXXXXXXX";
private static final String AUTH_TOKEN = "999aa999aaa999aaaa999";
private static final String CALL_SID = "CA83837718818gdgdg";
private static final String TWILIO_RES_URL = "https://api.twilio.com/2010-04-01/Accounts";
private static final String REC_EXT = ".mp3";
private static final String RUTA_RECS = "C:/recursos/grabaciones/";
public static void main(String[] args) {
try {
Twilio.init(ACCOUNT_SID, AUTH_TOKEN);
RecordingReader recordingReader = Recording.reader();
recordingReader.setCallSid(CALL_SID);
ResourceSet<Recording> recordings = recordingReader.read();
String recordingSid;
String urlGrabacion;
String locGrabacion;
InputStream in;
for (Recording recording : recordings) {
recordingSid = recording.getSid();
urlGrabacion = TWILIO_RES_URL + "/" + ACCOUNT_SID + "/Recordings/" + recordingSid + REC_EXT;
locGrabacion = RUTA_RECS + CALL_SID + "_" + recordingSid + REC_EXT;
System.out.println("Recuperando grabacion " + recordingSid);
System.out.println("Ubicacion remota " + urlGrabacion);
if (!new File(RUTA_RECS).exists()) {
new File(RUTA_RECS).mkdirs();
}
in = new URL(urlGrabacion).openStream();
Files.copy(in, Paths.get(locGrabacion), StandardCopyOption.REPLACE_EXISTING);
System.out.println("Ubicacion local " + locGrabacion);
in.close();
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
Once you know the recordingSid for example RE557ce644e5ab84fa21cc21112e22c485
you can get a .wav file at https://api.twilio.com/2010-04-01/Accounts/ACXXXXX.../Recordings/RE557ce644e5ab84fa21cc21112e22c485.wav
You can get a .mp3 file at https://api.twilio.com/2010-04-01/Accounts/ACXXXXX.../Recordings/RE557ce644e5ab84fa21cc21112e22c485.mp3
where ACXXXXX... is your Twilio account SID (ACCOUNT_SID)
I am using java spark API to write some test application . I am using a class which doesn't extends serializable interface . So to make the application work I am using kryo serializer to serialize the class . But the problem which I observed while debugging was that during the de-serialization the returned class object becomes null and in turn throws a null pointer exception . It seems to be closure problem where things are going wrong but not sure.Since I am new to this kind of serialization I don't know where to start digging.
Here is the code I am testing :
package org.apache.spark.examples;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.InetAddress;
import java.net.UnknownHostException;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
/**
* Spark application to test the Serialization issue in spark
*/
public class Test {
static PrintWriter outputFileWriter;
static FileWriter file;
static JavaSparkContext ssc;
public static void main(String[] args) {
String inputFile = "/home/incubator-spark/examples/src/main/scala/org/apache/spark/examples/InputFile.txt";
String master = "local";
String jobName = "TestSerialization";
String sparkHome = "/home/test/Spark_Installation/spark-0.7.0";
String sparkJar = "/home/test/TestSerializationIssesInSpark/TestSparkSerIssueApp/target/TestSparkSerIssueApp-0.0.1-SNAPSHOT.jar";
SparkConf conf = new SparkConf();
conf.set("spark.closure.serializer","org.apache.spark.serializer.KryoSerializer");
conf.set("spark.kryo.registrator", "org.apache.spark.examples.MyRegistrator");
// create the Spark context
if(master.equals("local")){
ssc = new JavaSparkContext("local", jobName,conf);
//ssc = new JavaSparkContext("local", jobName);
} else {
ssc = new JavaSparkContext(master, jobName, sparkHome, sparkJar);
}
JavaRDD<String> testData = ssc.textFile(inputFile).cache();
final NotSerializableJavaClass notSerializableTestObject= new NotSerializableJavaClass("Hi ");
#SuppressWarnings({ "serial", "unchecked"})
JavaRDD<String> classificationResults = testData.map(
new Function<String, String>() {
#Override
public String call(String inputRecord) throws Exception {
if(!inputRecord.isEmpty()) {
//String[] pointDimensions = inputRecord.split(",");
String result = "";
try {
FileWriter file = new FileWriter("/home/test/TestSerializationIssesInSpark/results/test_result_" + (int) (Math.random() * 100));
PrintWriter outputFile = new PrintWriter(file);
InetAddress ip;
ip = InetAddress.getLocalHost();
outputFile.println("IP of the server: " + ip);
result = notSerializableTestObject.testMethod(inputRecord);
outputFile.println("Result: " + result);
outputFile.flush();
outputFile.close();
file.close();
} catch (UnknownHostException e) {
e.printStackTrace();
}
catch (IOException e1) {
e1.printStackTrace();
}
return result;
} else {
System.out.println("End of elements in the stream.");
String result = "End of elements in the input data";
return result;
}
}
}).cache();
long processedRecords = classificationResults.count();
ssc.stop();
System.out.println("sssssssssss"+processedRecords);
}
}
Here is the KryoRegistrator class
package org.apache.spark.examples;
import org.apache.spark.serializer.KryoRegistrator;
import com.esotericsoftware.kryo.Kryo;
public class MyRegistrator implements KryoRegistrator {
public void registerClasses(Kryo kryo) {
kryo.register(NotSerializableJavaClass.class);
}
}
Here is the class I am serializing :
package org.apache.spark.examples;
public class NotSerializableJavaClass {
public String testVariable;
public NotSerializableJavaClass(String testVariable) {
super();
this.testVariable = testVariable;
}
public String testMethod(String vartoAppend){
return this.testVariable + vartoAppend;
}
}
This is because spark.closure.serializer only supports the Java serializer. See http://spark.apache.org/docs/latest/configuration.html about spark.closure.serializer
In my quest to create a simple Java program to extract tweets from Twitter's streaming API, I have modified this (http://cotdp.com/dl/TwitterConsumer.java) code snippet to work with the OAuth method. The result is the below code, which when executed, throws a Connection Refused Exception.
I am aware of Twitter4J however I want to create a program that relies least on other APIs.
I have done my research and it looks like the oauth.signpost library is suitable for Twitter's streaming API. I have also ensured my authentication details are correct. My Twitter Access level is 'Read-only'.
I couldn't find a simple Java example that shows how to use the streaming API without relying on e.g. Twitter4j.
import java.io.BufferedReader;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import oauth.signpost.OAuthConsumer;
import oauth.signpost.commonshttp.CommonsHttpOAuthConsumer;
/**
* A hacky little class illustrating how to receive and store Twitter streams
* for later analysis, requires Apache Commons HTTP Client 4+. Stores the data
* in 64MB long JSON files.
*
* Usage:
*
* TwitterConsumer t = new TwitterConsumer("username", "password",
* "http://stream.twitter.com/1/statuses/sample.json", "sample");
* t.start();
*/
public class TwitterConsumer extends Thread {
//
static String STORAGE_DIR = "/tmp";
static long BYTES_PER_FILE = 64 * 1024 * 1024;
//
public long Messages = 0;
public long Bytes = 0;
public long Timestamp = 0;
private String accessToken = "";
private String accessSecret = "";
private String consumerKey = "";
private String consumerSecret = "";
private String feedUrl;
private String filePrefix;
boolean isRunning = true;
File file = null;
FileWriter fw = null;
long bytesWritten = 0;
public static void main(String[] args) {
TwitterConsumer t = new TwitterConsumer(
"XXX",
"XXX",
"XXX",
"XXX",
"http://stream.twitter.com/1/statuses/sample.json", "sample");
t.start();
}
public TwitterConsumer(String accessToken, String accessSecret, String consumerKey, String consumerSecret, String url, String prefix) {
this.accessToken = accessToken;
this.accessSecret = accessSecret;
this.consumerKey = consumerKey;
this.consumerSecret = consumerSecret;
feedUrl = url;
filePrefix = prefix;
Timestamp = System.currentTimeMillis();
}
/**
* #throws IOException
*/
private void rotateFile() throws IOException {
// Handle the existing file
if (fw != null)
fw.close();
// Create the next file
file = new File(STORAGE_DIR, filePrefix + "-"
+ System.currentTimeMillis() + ".json");
bytesWritten = 0;
fw = new FileWriter(file);
System.out.println("Writing to " + file.getAbsolutePath());
}
/**
* #see java.lang.Thread#run()
*/
public void run() {
// Open the initial file
try { rotateFile(); } catch (IOException e) { e.printStackTrace(); return; }
// Run loop
while (isRunning) {
try {
OAuthConsumer consumer = new CommonsHttpOAuthConsumer(consumerKey, consumerSecret);
consumer.setTokenWithSecret(accessToken, accessSecret);
HttpGet request = new HttpGet(feedUrl);
consumer.sign(request);
DefaultHttpClient client = new DefaultHttpClient();
HttpResponse response = client.execute(request);
BufferedReader reader = new BufferedReader(
new InputStreamReader(response.getEntity().getContent()));
while (true) {
String line = reader.readLine();
if (line == null)
break;
if (line.length() > 0) {
if (bytesWritten + line.length() + 1 > BYTES_PER_FILE)
rotateFile();
fw.write(line + "\n");
bytesWritten += line.length() + 1;
Messages++;
Bytes += line.length() + 1;
}
}
} catch (Exception e) {
e.printStackTrace();
}
System.out.println("Sleeping before reconnect...");
try { Thread.sleep(15000); } catch (Exception e) { }
}
}
}
}
I tried to simulate the code and found that the error was very simple. You should use https instead of http in the url :)
Look at the test code I have written below.
Using pure java I set an Authenticator and make a URI call to get some xml data and convert it to an object.
I wrote the code below to test performance of hotpotato (netty) vs. pure java (no pipelining).
The trouble is, I can't figure out how to Authenticate my request with hotpotato or netty, code for either is acceptable, I just want to test the performance diff (i.e. see how many requests will be performed in 5 seconds).
public static void main(String[] args) throws Exception {
Authenticator.setDefault(new MyAuthenticator("DummyUser", "DummyPassword"));
int timeToTestFor = 5000; //5 seconds;
int count = 0;
System.out.println("Start time");
long starttime = System.currentTimeMillis();
do {
URL url = new URL(
"http://example.com/rest/GetData.ashx?what=pizza&where=new%20york&visitorId=12345&sessionId=123456");
SearchResultsDocument doc = SearchResultsDocument.Factory.parse(url);
count++;
} while (System.currentTimeMillis() - starttime < timeToTestFor);
System.out.println("DONE Total count=" + count);
System.out.println("Netty/Hotpotatoe Start time");
count = 0;
starttime = System.currentTimeMillis();
do {
// Create & initialise the client
HttpClient client = new DefaultHttpClient();
client.init();
// Setup the request
HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_0,
HttpMethod.GET, "/rest/GetData.ashx?what=pizza&where=new%20york&visitorId=12345&sessionId=123456");
// Execute the request, turning the result into a String
HttpRequestFuture future = client.execute("example.com", 80, request,
new BodyAsStringProcessor());
future.awaitUninterruptibly();
// Print some details about the request
System.out.println("A >> " + future);
// If response was >= 200 and <= 299, print the body
if (future.isSuccessfulResponse()) {
System.out.println("B >> "+future.getProcessedResult());
}
// Cleanup
client.terminate();
count++;
} while (System.currentTimeMillis() - starttime < timeToTestFor);
System.out.println("DONE Total count=" + count);
}
Here is working example of using basic authentication with Netty only. Tested with Jetty as a server requiring basic authentication.
import java.net.InetSocketAddress;
import java.util.concurrent.Executors;
import org.jboss.netty.bootstrap.ClientBootstrap;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffers;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.ChannelPipeline;
import org.jboss.netty.channel.ChannelPipelineFactory;
import org.jboss.netty.channel.Channels;
import org.jboss.netty.channel.ExceptionEvent;
import org.jboss.netty.channel.MessageEvent;
import org.jboss.netty.channel.SimpleChannelHandler;
import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory;
import org.jboss.netty.handler.codec.base64.Base64;
import org.jboss.netty.handler.codec.http.DefaultHttpRequest;
import org.jboss.netty.handler.codec.http.HttpChunkAggregator;
import org.jboss.netty.handler.codec.http.HttpClientCodec;
import org.jboss.netty.handler.codec.http.HttpHeaders;
import org.jboss.netty.handler.codec.http.HttpMethod;
import org.jboss.netty.handler.codec.http.HttpResponse;
import org.jboss.netty.handler.codec.http.HttpVersion;
import org.jboss.netty.util.CharsetUtil;
public class BasicAuthTest {
private static final int PORT = 80;
private static final String USERNAME = "";
private static final String PASSWORD = "";
private static final String URI = "";
private static final String HOST = "";
public static void main(String[] args) {
ClientBootstrap client = new ClientBootstrap(
new NioClientSocketChannelFactory(
Executors.newCachedThreadPool(),
Executors.newCachedThreadPool()));
client.setPipelineFactory(new ChannelPipelineFactory() {
#Override
public ChannelPipeline getPipeline() throws Exception {
ChannelPipeline pipeline = Channels.pipeline();
pipeline.addLast("codec", new HttpClientCodec());
pipeline.addLast("aggregator", new HttpChunkAggregator(5242880));
pipeline.addLast("authHandler", new ClientMessageHandler());
return pipeline;
}
});
DefaultHttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, URI);
request.addHeader(HttpHeaders.Names.HOST, HOST);
String authString = USERNAME + ":" + PASSWORD;
ChannelBuffer authChannelBuffer = ChannelBuffers.copiedBuffer(authString, CharsetUtil.UTF_8);
ChannelBuffer encodedAuthChannelBuffer = Base64.encode(authChannelBuffer);
request.addHeader(HttpHeaders.Names.AUTHORIZATION, encodedAuthChannelBuffer.toString(CharsetUtil.UTF_8));
client.connect(new InetSocketAddress(HOST, PORT)).awaitUninterruptibly().getChannel()
.write(request).awaitUninterruptibly();
}
public static class ClientMessageHandler extends SimpleChannelHandler {
#Override
public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
e.getCause().printStackTrace();
}
#Override
public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
HttpResponse httpResponse = (HttpResponse) e.getMessage();
String json = httpResponse.getContent().toString(CharsetUtil.UTF_8);
System.out.println(json);
}
}
}