Related
Context
I am working on my Master Thesis using Intel SGX in order to use Enclave Code alongside a Java project.
The project uses Gradle for installation, and I have added a .jar containing a Class containing Native calls and the .so file containing the code for these calls.
I am currently only being able to open up an enclave and calculate an EC Curve for DH Key exchange, however, as the java code moves on, segfaults occur. These Segfaults however, dont occur inside the C code, as I am able to retrieve the correct and expected results from the native code in my java project. These segmentation faults usually happen there is heavy I/O workload (At least that is what I deduced so far), usually Sockets, which is something I use, since my thesis is in Distributed Systems.
C Code:
JNIEXPORT jint JNICALL Java_sgxUtils_SgxFunctions_jni_1initialize_1enclave
(JNIEnv *env , jobject,jint enclaveId, jbyteArray file_path){
if(file_path == NULL){
printf("NULL FILEPATH. error.");
return -1;
}
char token_path[MAX_PATH] = {'\0'};
sgx_launch_token_t token = {0};
sgx_status_t ret = SGX_ERROR_UNEXPECTED;
int updated = 0;
//Get the id of a given Enclave:
global_eid = (int)enclaveId;
int length = snprintf(NULL , 0, "%d", enclaveId) + 1; //Gives the length of the Integer as a String.
char* id_ptr = (char*)malloc(length);
snprintf(id_ptr,length,"%d",enclaveId);
/* Step 1: try to retrieve the launch token saved by last transaction
* if there is no token, then create a new one.
*/
/* try to get the token saved in $HOME */
const char *home_dir = getpwuid(getuid())->pw_dir;
if (home_dir != NULL &&
(strlen(home_dir)+strlen("/")+sizeof(TOKEN_FILENAME)+1) <= MAX_PATH) {
/* compose the token path */
strncpy(token_path, home_dir, strlen(home_dir));
strncat(token_path, "/", strlen("/"));
char* correctName = (char*) malloc(length + sizeof(TOKEN_FILENAME) + 1);
memcpy(correctName,id_ptr,length);
strncat(correctName,TOKEN_FILENAME,sizeof(TOKEN_FILENAME)+1);
strncat(token_path,correctName,length + sizeof(TOKEN_FILENAME)+1);
// strncat(token_path, TOKEN_FILENAME, sizeof(TOKEN_FILENAME)+1);
// free(correctName);
} else {
/* if token path is too long or $HOME is NULL */
strncpy(token_path, TOKEN_FILENAME, sizeof(TOKEN_FILENAME));
}
FILE *fp = fopen(token_path, "rb");
if (fp == NULL && (fp = fopen(token_path, "wb")) == NULL) {
printf("Warning: Failed to create/open the launch token file \"%s\".\n", token_path);
}
if (fp != NULL) {
/* read the token from saved file */
size_t read_num = fread(token, 1, sizeof(sgx_launch_token_t), fp);
if (read_num != 0 && read_num != sizeof(sgx_launch_token_t)) {
/* if token is invalid, clear the buffer */
memset(&token, 0x0, sizeof(sgx_launch_token_t));
printf("Warning: Invalid launch token read from \"%s\".\n", token_path);
}
}
printf("\n");
/* Step 2: call sgx_create_enclave to initialize an enclave instance */
/* Debug Support: set 2nd parameter to 1 */
//Here we obtain the char* of the given Enclave file path:
size_t filename_size = env->GetArrayLength(file_path);
jbyte* jb_filePath = env->GetByteArrayElements(file_path,NULL);
char* enclave_filePath = (char*) malloc(filename_size);
memcpy(enclave_filePath,jb_filePath,filename_size);
ret = sgx_create_enclave(enclave_filePath, SGX_DEBUG_FLAG, &token, &updated, &global_eid, NULL);
if (ret != SGX_SUCCESS) {
if (fp != NULL) fclose(fp);
return -1;
}
/* Step 3: save the launch token if it is updated */
if (updated == FALSE || fp == NULL) {
/* if the token is not updated, or file handler is invalid, do not perform saving */
if (fp != NULL) fclose(fp);
return 0;
}
/* reopen the file with write capablity */
fp = freopen(token_path, "wb", fp);
if (fp == NULL) return 0;
size_t write_num = fwrite(token, 1, sizeof(sgx_launch_token_t), fp);
if (write_num != sizeof(sgx_launch_token_t))
printf("Warning: Failed to save launch token to \"%s\".\n", token_path);
fclose(fp);
free(id_ptr);
// env->ReleaseStringUTFChars(file_path,enclave_filePath);
return 0;
}
JNIEXPORT jbyteArray JNICALL Java_sgxUtils_SgxFunctions_jni_1sgx_1begin_1ec_1dh(JNIEnv *env, jobject){
printf("Begin DH \n");
if(init_happened == 1){ //Verify if the Random generators were seeded.
sgx_status_t init_status;
init_mbed(global_eid,&init_status);
init_happened = 0;
//printf("Init happened inside enclave.");
}
unsigned char* public_ec_key = (unsigned char*)malloc(EC_KEY_SIZE); //32 bytes allocated. (256)
printf("Calculating EC_DH.\n");
//Start the ECDH Ecall.
sgx_status_t ecdh_status;
create_ec_key_pair(global_eid,&ecdh_status,public_ec_key,EC_KEY_SIZE); //Send the buffer and its size.
if(ecdh_status != SGX_SUCCESS){
printf("Failed. ec_key_pair returned %d\n",ecdh_status);
}
//Copy the information to a Jbyte in order to return in a JbyteArray:
jbyte* toCopy = (jbyte*)malloc(EC_KEY_SIZE * sizeof(jbyte));
memcpy(toCopy,public_ec_key,EC_KEY_SIZE);
//Return the Public Key:
jbyteArray result = env->NewByteArray(EC_KEY_SIZE);
env->SetByteArrayRegion(result,0,EC_KEY_SIZE,toCopy);
//Free the pointers and return the arrray.
free(public_ec_key);
free(toCopy);
printf("EC_DH Calculated.\n");
return result;
}
I will omit the large output from Valgrind, it is just a huge file with '?' every line, since SGX hides the Stack trace, giving me no info on the invalid reads/Writes.
Valgrind Output
==38490== Warning: client switching stacks? SP change: 0x6a937a0 --> 0x28789d48
==38490== to suppress, use: --max-stackframe=567240104 or greater
==38490== Warning: client switching stacks? SP change: 0x28789d48 --> 0x6a937a0
==38490== to suppress, use: --max-stackframe=567240104 or greater
==38490== Warning: client switching stacks? SP change: 0x6a97220 --> 0x28789d48
==38490== to suppress, use: --max-stackframe=567225128 or greater
==38490== further instances of this message will not be shown.
==38490==
==38490== Process terminating with default action of signal 11 (SIGSEGV)
==38490== Access not within mapped region at address 0x2
==38490== at 0x27CD0B05: sig_handler_sim(int, siginfo_t*, void*) (in /opt/intel/sgxsdk/lib64/libsgx_urts_sim.so)
==38490== If you believe this happened as a result of a stack
==38490== overflow in your program's main thread (unlikely but
==38490== possible), you can try to increase the size of the
==38490== main thread stack using the --main-stacksize= flag.
==38490== The main thread stack size used in this run was 8388608.
==38490==
==38490== HEAP SUMMARY:
==38490== in use at exit: 16,636,139 bytes in 39,334 blocks
==38490== total heap usage: 55,264 allocs, 15,930 frees, 71,819,001 bytes allocated
Java Code
private void initEnclave() {
this.logger.info("Starting InitEnclave");
enclave = new SgxFunctions(this.enclaveId);
try {
File pemFile = new File(HOME_DIR + "/" + this.enclaveId + ".pem");
if(!pemFile.exists()) {
pemFile = SgxFunctions.createPem(this.enclaveId);
}
File enclaveFile = new File(HOME_DIR + "/" + this.enclaveId + "enclave_signed.so");
if(!enclaveFile.exists()) {
enclave.createSignedEnclave(HOME_DIR, pemFile.getAbsolutePath(), enclaveId);
this.logger.info("Enclave signed");
}
else
this.logger.info("Enclave file exists. Starting enclave.");
logger.info("ABSOLUTE PATH: " + enclaveFile.getAbsolutePath());
byte[] enclaveFilePath = enclaveFile.getAbsolutePath().substring(0).getBytes(StandardCharsets.UTF_8);
byte[] correctPath = Arrays.copyOf(enclaveFilePath, enclaveFilePath.length+1);
correctPath[correctPath.length-1] = '\0';
int created = enclave.jni_initialize_enclave(enclaveId,correctPath);
if(created == 0)
this.logger.info("Enclave created.");
else
this.logger.info("Enclave failed to create.");
//
// //Create DH parameters:
this.dh_params = this.enclave.jni_sgx_begin_ec_dh(); //Begin the Enclave EC parameters for DH key exchange.
this.logger.info("EC-DH Parameters created for Key exchange.");
} catch (Exception e) {
e.printStackTrace();
}
}
If I comment out the 'begin_ec_dh()' Call, the program will not segfault. However, even when running this native call, the Enclave will return an acceptable result and I am able to use the result in my Java code no problem, but only up until a certain point where it segfaults for an unkown reason.
I suspect its obviously my code, but I have no idea what is going on.
Any help is appreciated, Thank you for your help and sorry for the long post.
Edit:
I have added a printf() in order to retrieve a Crash file from the JVM over the filename Pointer.
What I dont understand is that If you look at my C code, I have allocated exactly the Array Size that comes from java (And I have added the null terminator in my Java Code, which was previously the source of another segfault), any idea what is causing this? Seems like I am dereferencing an Invalid pointer, but why is it invalid?
JVM Crash File
# A fatal error has been detected by the Java Runtime Environment:
#
# SIGSEGV (0xb) at pc=0x00007f9f51fd9627, pid=9910, tid=9911
#
# JRE version: OpenJDK Runtime Environment (17.0.3+7) (build 17.0.3+7-Ubuntu-0ubuntu0.20.04.1)
# Java VM: OpenJDK 64-Bit Server VM (17.0.3+7-Ubuntu-0ubuntu0.20.04.1, mixed mode, sharing, tiered, compressed oops, compressed class ptrs, g1 gc, linux-amd64)
# Problematic frame:
# C [libc.so.6+0x188627]
#
# Core dump will be written. Default location: Core dumps may be processed with "/usr/share/apport/apport -p%p -s%s -c%c -d%d -P%P -u%u -g%g -- %E" (or dumping to /home/andrecruz/Tese/Con-BFT/build/install/library/core.9910)
#
# If you would like to submit a bug report, please visit:
# Unknown
# The crash happened outside the Java Virtual Machine in native code.
# See problematic frame for where to report the bug.
#
--------------- S U M M A R Y ------------
Command Line: -Djava.security.properties=./config/java.security -Djava.library.path=./lib/ -Dfile.encoding=UTF-8 -XX:ErrorFile=/home/andrecruz/Tese/Con-BFT/ -XX:+ShowCodeDetailsInExceptionMessages -Dlogback.configurationFile=./config/logback.xml bftsmart.demo.map.MapServer 0 10
Host: Intel(R) Core(TM) i5-6440HQ CPU # 2.60GHz, 4 cores, 7G, Ubuntu 20.04.4 LTS
Time: Sun May 29 23:13:08 2022 WEST elapsed time: 2.134101 seconds (0d 0h 0m 2s)
--------------- T H R E A D ---------------
Current thread (0x00007f9f4c012b40): JavaThread "main" [_thread_in_native, id=9911, stack(0x00007f9f5061b000,0x00007f9f5071c000)]
Stack: [0x00007f9f5061b000,0x00007f9f5071c000], sp=0x00007f9f50718c28, free space=1015k
Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)
C [libc.so.6+0x188627]
C [libc.so.6+0x61d6f] _IO_printf+0xaf
j sgxUtils.SgxFunctions.jni_initialize_enclave(I[B)I+0
j bftsmart.tom.ServiceReplica.initEnclave()V+248
j bftsmart.tom.ServiceReplica.<init>(ILjava/lang/String;Lbftsmart/tom/server/Executable;Lbftsmart/tom/server/Recoverable;Lbftsmart/tom/server/RequestVerifier;Lbftsmart/tom/server/Replier;Lbftsmart/tom/util/KeyLoader;I)V+147
j bftsmart.tom.ServiceReplica.<init>(ILbftsmart/tom/server/Executable;Lbftsmart/tom/server/Recoverable;I)V+17
j bftsmart.demo.map.MapServer.<init>(II)V+35
j bftsmart.demo.map.MapServer.main([Ljava/lang/String;)V+34
v ~StubRoutines::call_stub
V [libjvm.so+0x83ae75]
V [libjvm.so+0x8cf4e9]
V [libjvm.so+0x8d2392]
C [libjli.so+0x4abe]
C [libjli.so+0x84ed]
Java frames: (J=compiled Java code, j=interpreted, Vv=VM code)
j sgxUtils.SgxFunctions.jni_initialize_enclave(I[B)I+0
j bftsmart.tom.ServiceReplica.initEnclave()V+248
j bftsmart.tom.ServiceReplica.<init>(ILjava/lang/String;Lbftsmart/tom/server/Executable;Lbftsmart/tom/server/Recoverable;Lbftsmart/tom/server/RequestVerifier;Lbftsmart/tom/server/Replier;Lbftsmart/tom/util/KeyLoader;I)V+147
j bftsmart.tom.ServiceReplica.<init>(ILbftsmart/tom/server/Executable;Lbftsmart/tom/server/Recoverable;I)V+17
j bftsmart.demo.map.MapServer.<init>(II)V+35
j bftsmart.demo.map.MapServer.main([Ljava/lang/String;)V+34
v ~StubRoutines::call_stub
siginfo: si_signo: 11 (SIGSEGV), si_code: 1 (SEGV_MAPERR), si_addr: 0x0000000000000020
Register to memory mapping:
RAX=0x0000000000000010 is an unknown value
RBX=0x00007f9f51ec7b43: <offset 0x0000000000076b43> in /lib/x86_64-linux-gnu/libc.so.6 at 0x00007f9f51e51000
RCX=0x000000000000000f is an unknown value
RDX=0x000000000000002f is an unknown value
RSP=0x00007f9f50718c28 is pointing into the stack for thread: 0x00007f9f4c012b40
RBP=0x00007f9f507191a0 is pointing into the stack for thread: 0x00007f9f4c012b40
RSI=0x00007f9f50719168 is pointing into the stack for thread: 0x00007f9f4c012b40
RDI=0x0000000000000020 is an unknown value
R8 =0x00000000ffffffff points into unknown readable memory: 00
R9 =0x0000000000000009 is an unknown value
R10=0x00007f9f50610099: <offset 0x0000000000007099> in /home/andrecruz/Tese/Con-BFT/build/install/library/lib/libSgx.so at 0x00007f9f50609000
R11=0x000000000000002f is an unknown value
R12=0x00007f9f5203e6a0: _IO_2_1_stdout_+0x0000000000000000 in /lib/x86_64-linux-gnu/libc.so.6 at 0x00007f9f51e51000
R13=0x00007f9f5061008f: <offset 0x000000000000708f> in /home/andrecruz/Tese/Con-BFT/build/install/library/lib/libSgx.so at 0x00007f9f50609000
R14=0x00007f9f507191b0 is pointing into the stack for thread: 0x00007f9f4c012b40
R15=0x0000000000000073 is an unknown value
I am using the Android NDK on a Motorola G6, running Android 8.0.
I am using JNI to perform periodic callbacks to my Java code in the following:
if ( data_arg && g_vm && g_obj ) {
jint res = (*g_vm)->GetEnv(g_vm, (void**)&g_env, JNI_VERSION_1_6);
if (res != JNI_OK) {
res = (*g_vm)->AttachCurrentThread(g_vm, &g_env, NULL);
if (JNI_OK != res) {
return NULL;
}
jclass interfaceClass = (*g_env)->GetObjectClass(g_env,g_obj);
if (!interfaceClass) {
return NULL;
}
g_mid = (*g_env)->GetMethodID(g_env,interfaceClass, "readCallback", "(I[B)V");
if (!g_mid){
return NULL;
}
jbyteArray dataByteArray = (*g_env) -> NewByteArray(g_env,data_arg->len);
(*g_env)->SetByteArrayRegion(g_env, dataByteArray, 0,data_arg->len,(jbyte*)data_arg->data);
(*g_env)->CallVoidMethod(g_env, g_obj,(jmethodID)g_mid, (jint)data_arg->len, (jbyteArray)dataByteArray);
(*g_env)->DeleteLocalRef(g_env,dataByteArray);
(*g_env)->DeleteLocalRef(g_env,res);
(*g_env)->DeleteLocalRef(g_env,interfaceClass);
}
The g_vm and g_obj variables are initialized when the app starts up with the following:
JNIEXPORT jint JNICALL Java_com_abc_register(JNIEnv *env, jobject obj) {
int returnValueBool = 1;
(*env)->GetJavaVM(env, &g_vm);
g_obj = (*env)->NewGlobalRef(env,obj);
g_env = env;
g_res = JNI_ERR;
return (jint)returnValueBool;
}
The class that calls the above register method is a static instance.
The callbacks occur about about 20 Hz. The callbacks are called from a single thread, which is why the DetachCurrentThread method is not called. The app typically runs ok for a few minutes, until it crashes.
The crash produces the following stack trace:
* thread #65, name = 'JVMTI Agent thr', stop reason = signal SIGSEGV: invalid address (fault address: 0x765630)
* frame #0: 0x92b8d44e libopenjdkjvmti.so`openjdkjvmti::MethodUtil::GetMethodName(_jvmtiEnv*, _jmethodID*, char**, char**, char**) + 410
Before the crash occurs, memory and cpu usage are not excessive, nor is the thread count too high. I do not believe there is a memory leak based on the android profiler view
shown here
Thanks a ton in advance!
I need to add a timeout for the reply/request transaction using 0MQ. How is this typically accomplished? I tried using the method :
socket.setReceiveTimeOut();
and
socket.setSendTimeout();
but they seem to cause a null pointer exception.
In essence, I want the application to timeout after 10 seconds if the application receiving the request is not available.
Any help is appreciated.
Thanks!
I think jzmq should throw a ZMQException when recv timeout,
But there is no ZMQException, when err = EAGAIN.
https://github.com/zeromq/jzmq/blob/master/jzmq-jni/src/main/c%2B%2B/Socket.cpp
static
zmq_msg_t *do_read(JNIEnv *env, jobject obj, zmq_msg_t *message, int flags)
{
void *socket = get_socket (env, obj);
int rc = zmq_msg_init (message);
if (rc != 0) {
raise_exception (env, zmq_errno());
return NULL;
}
#if ZMQ_VERSION >= ZMQ_MAKE_VERSION(3,0,0)
rc = zmq_recvmsg (socket, message, flags);
#else
rc = zmq_recv (socket, message, flags);
#endif
int err = zmq_errno();
if (rc < 0 && err == EAGAIN) {
rc = zmq_msg_close (message);
err = zmq_errno();
if (rc != 0) {
raise_exception (env, err);
return NULL;
}
return NULL;
}
if (rc < 0) {
raise_exception (env, err);
rc = zmq_msg_close (message);
err = zmq_errno();
if (rc != 0) {
raise_exception (env, err);
return NULL;
}
return NULL;
}
return message;
}
I wonder if your null pointer is related to how your socket was created. I have set a socket timeout successfully in the past.
The following has worked for me when I used the JeroMQ library (native Java implementation of ZMQ). I used this to help do REQ-REP commands via ZMQ.
ZMQ.Context context = ZMQ.context(1);
ZMQ.Socket sock = context.socket(ZMQ.REQ);
sock.setSendTimeOut(10000); // 10 second send timeout
sock.setReceiveTimeOut(10000); // 10 second receive timeout
if (sock.connect("tcp://127.0.0.1:1234")) {
if (sock.send(/* insert data here */)) {
/* Send was successful and did not time out. */
byte[] replyBytes = null;
replyBytes = sock.recv();
if (null == replyBytes) {
/* Receive timed out. */
} else {
/* Receive was successful. Do something with replyBytes. */
}
}
}
How is this [a timeout for the reply/request transaction] typically accomplished ?
I am sad to confirm, there is nothing like this in the ZeroMQ native API. The principle of doing async delivery means, there is no limit for delivery to take place ( in a best-effort model of scheduling, or not at all ).
If new to the ZeroMQ, you may enjoy a fast read into this 5-second read about the main conceptual elements in [ ZeroMQ hierarchy in less than a five seconds ] Section.
I want ... to timeout after 10 seconds if ... receiving the request is not ...
May design your .recv()-method call usage into either a pre-tested / protected after a .poll( 10000 )-method screener, to first explicitly detect a presence of any message for indeed being delivered to your application-code, before ever issuing ( or not ) a call to the actual .recv()-method only upon a previously POSACK-ed message to be ready to get locally read, or may use a bit more "raw" approach, using a handler with a non-blocking form of the method, by a call to the .recv( ZMQ_NOBLOCK )-flagged not to spend a millisecond "there", in cases when "there" are no messages to read right now from the local-side Context()-engine instance, and handle each of the cases accordingly in your code.
A Bonus Point
Also be warned, that using the REQ/REP-Scalable Formal Communication Archetype pattern will not be any easier, as there is a mandatory two-side-step-dance ( sure, if not intentionally artificially ZMQ_RELAXED ), so the both FSA-back-to-back-connected-FSA-s will still have to wait for the next "expected" remote-event, before becoming able to make a chance for handling the next local-event. If interested in details, one will find many posts on un-avoidable, un-salvagable mutual-deadlock, that is sure to happen for REQ/REP, where we only do not know when it happens, but are sure it will.
I'd like to monitor Informatica ETL Workflows from custom Java program, via informatica Development Platform (LMapi), v9.1
I already have got C program, it works fine, but it would be great port to Java.
We get a lot of C DLLs, with asynchronous functions e.g.: JavaLMApi.dll, INFA_LMMonitorServerA (with detailed event-log possibilities)
In header we can see:
PMLM_API_EXTSPEC INFA_API_STATUS
INFA_LMMonitorServerA
(
INFA_UINT32 connectionId,
struct INFA_LMAPI_MONITOR_SERVER_REQUEST_PARAMS *request,
void *clientContext,
INFA_UINT32 *requestId
);
There is no any documentation for this problem, only with this information should I use for soulution.
The question is: how to call/use INFA_LMMonitorServerA in Java? (Dll-load with JNA/JNI is not problem, only the callback).
static INFA_UINT32 nConnectionId = 0;
/* C-"skeleton": */
void GetSrvDetailcallback(struct INFA_API_REPLY_CONTEXT* GetSrvDetailReplyCtxt)
{
INFA_API_STATUS apiRet;
struct INFA_LMAPI_SERVER_DETAILS *serverDetails = NULL;
char *serverStatus = NULL;
/* Check if the return status is Acknowledgement */
if (GetSrvDetailReplyCtxt->returnStatus == INFA_REQUEST_ACKNOWLEDGED)
{
fprintf(stdout, "\nINFA REQUEST ACKNOWLEDGED \n\n",NULL);
return;
}
apiRet = INFA_LMGetServerDetailsResults(GetSrvDetailReplyCtxt, &serverDetails);
/* Check the return code if if is an error */
if (INFA_SUCCESS != apiRet)
{
fprintf(stderr, "Error: INFA_LMGetServerDetailsResults returns %d\n", apiRet);
return;
}
printResults(serverDetails);
}
static void myServer()
{
struct INFA_LMAPI_CONNECT_SERVER_REQUEST_PARAMS_EX connectParamsex;
INFA_API_STATUS apiRet;
struct INFA_LMAPI_LOGIN_REQUEST_PARAMS loginparams;
apiRet = INFA_LMLogin(nConnectionId, &loginparams, NULL);
if (INFA_SUCCESS != apiRet)
{
fprintf(stderr, "Error: INFA_LMLogin returns %d\n", apiRet);
return;
}
struct INFA_LMAPI_MONITOR_SERVER_REQUEST_PARAMS strMonitorRequestParams;
//Only Running Tasks
strMonitorRequestParams.monitorMode = INFA_LMAPI_MONITOR_RUNNING;
strMonitorRequestParams.continuous = INFA_TRUE;
/* Get Server Details */
INFA_UINT32 GetSrvDetailsrequestId = 0;
/* Register a callback function. */
INFA_LMRegisterCallback(INFA_LMAPI_MONITOR_SERVER, &GetSrvDetailcallback);
apiRet = INFA_LMMonitorServerA(nConnectionId, &strMonitorRequestParams, NULL, &GetSrvDetailsrequestId);
if (INFA_SUCCESS != apiRet && INFA_REQUEST_PENDING != apiRet)
{
fprintf(stderr, "Error: INFA_LMMonitorServerA returns %d\n", apiRet);
return;
}
}
I'm trying to read / write multiple Protocol Buffers messages from files, in both C++ and Java. Google suggests writing length prefixes before the messages, but there's no way to do that by default (that I could see).
However, the Java API in version 2.1.0 received a set of "Delimited" I/O functions which apparently do that job:
parseDelimitedFrom
mergeDelimitedFrom
writeDelimitedTo
Are there C++ equivalents? And if not, what's the wire format for the size prefixes the Java API attaches, so I can parse those messages in C++?
Update:
These now exist in google/protobuf/util/delimited_message_util.h as of v3.3.0.
I'm a bit late to the party here, but the below implementations include some optimizations missing from the other answers and will not fail after 64MB of input (though it still enforces the 64MB limit on each individual message, just not on the whole stream).
(I am the author of the C++ and Java protobuf libraries, but I no longer work for Google. Sorry that this code never made it into the official lib. This is what it would look like if it had.)
bool writeDelimitedTo(
const google::protobuf::MessageLite& message,
google::protobuf::io::ZeroCopyOutputStream* rawOutput) {
// We create a new coded stream for each message. Don't worry, this is fast.
google::protobuf::io::CodedOutputStream output(rawOutput);
// Write the size.
const int size = message.ByteSize();
output.WriteVarint32(size);
uint8_t* buffer = output.GetDirectBufferForNBytesAndAdvance(size);
if (buffer != NULL) {
// Optimization: The message fits in one buffer, so use the faster
// direct-to-array serialization path.
message.SerializeWithCachedSizesToArray(buffer);
} else {
// Slightly-slower path when the message is multiple buffers.
message.SerializeWithCachedSizes(&output);
if (output.HadError()) return false;
}
return true;
}
bool readDelimitedFrom(
google::protobuf::io::ZeroCopyInputStream* rawInput,
google::protobuf::MessageLite* message) {
// We create a new coded stream for each message. Don't worry, this is fast,
// and it makes sure the 64MB total size limit is imposed per-message rather
// than on the whole stream. (See the CodedInputStream interface for more
// info on this limit.)
google::protobuf::io::CodedInputStream input(rawInput);
// Read the size.
uint32_t size;
if (!input.ReadVarint32(&size)) return false;
// Tell the stream not to read beyond that size.
google::protobuf::io::CodedInputStream::Limit limit =
input.PushLimit(size);
// Parse the message.
if (!message->MergeFromCodedStream(&input)) return false;
if (!input.ConsumedEntireMessage()) return false;
// Release the limit.
input.PopLimit(limit);
return true;
}
Okay, so I haven't been able to find top-level C++ functions implementing what I need, but some spelunking through the Java API reference turned up the following, inside the MessageLite interface:
void writeDelimitedTo(OutputStream output)
/* Like writeTo(OutputStream), but writes the size of
the message as a varint before writing the data. */
So the Java size prefix is a (Protocol Buffers) varint!
Armed with that information, I went digging through the C++ API and found the CodedStream header, which has these:
bool CodedInputStream::ReadVarint32(uint32 * value)
void CodedOutputStream::WriteVarint32(uint32 value)
Using those, I should be able to roll my own C++ functions that do the job.
They should really add this to the main Message API though; it's missing functionality considering Java has it, and so does Marc Gravell's excellent protobuf-net C# port (via SerializeWithLengthPrefix and DeserializeWithLengthPrefix).
I solved the same problem using CodedOutputStream/ArrayOutputStream to write the message (with the size) and CodedInputStream/ArrayInputStream to read the message (with the size).
For example, the following pseudo-code writes the message size following by the message:
const unsigned bufLength = 256;
unsigned char buffer[bufLength];
Message protoMessage;
google::protobuf::io::ArrayOutputStream arrayOutput(buffer, bufLength);
google::protobuf::io::CodedOutputStream codedOutput(&arrayOutput);
codedOutput.WriteLittleEndian32(protoMessage.ByteSize());
protoMessage.SerializeToCodedStream(&codedOutput);
When writing you should also check that your buffer is large enough to fit the message (including the size). And when reading, you should check that your buffer contains a whole message (including the size).
It definitely would be handy if they added convenience methods to C++ API similar to those provided by the Java API.
IsteamInputStream is very fragile to eofs and other errors that easily occurs when used together with std::istream. After this the protobuf streams are permamently damaged and any already used buffer data is destroyed. There are proper support for reading from traditional streams in protobuf.
Implement google::protobuf::io::CopyingInputStream and use that together with CopyingInputStreamAdapter. Do the same for the output variants.
In practice a parsing call ends up in google::protobuf::io::CopyingInputStream::Read(void* buffer, int size) where a buffer is given. The only thing left to do is read into it somehow.
Here's an example for use with Asio synchronized streams (SyncReadStream/SyncWriteStream):
#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
using namespace google::protobuf::io;
template <typename SyncReadStream>
class AsioInputStream : public CopyingInputStream {
public:
AsioInputStream(SyncReadStream& sock);
int Read(void* buffer, int size);
private:
SyncReadStream& m_Socket;
};
template <typename SyncReadStream>
AsioInputStream<SyncReadStream>::AsioInputStream(SyncReadStream& sock) :
m_Socket(sock) {}
template <typename SyncReadStream>
int
AsioInputStream<SyncReadStream>::Read(void* buffer, int size)
{
std::size_t bytes_read;
boost::system::error_code ec;
bytes_read = m_Socket.read_some(boost::asio::buffer(buffer, size), ec);
if(!ec) {
return bytes_read;
} else if (ec == boost::asio::error::eof) {
return 0;
} else {
return -1;
}
}
template <typename SyncWriteStream>
class AsioOutputStream : public CopyingOutputStream {
public:
AsioOutputStream(SyncWriteStream& sock);
bool Write(const void* buffer, int size);
private:
SyncWriteStream& m_Socket;
};
template <typename SyncWriteStream>
AsioOutputStream<SyncWriteStream>::AsioOutputStream(SyncWriteStream& sock) :
m_Socket(sock) {}
template <typename SyncWriteStream>
bool
AsioOutputStream<SyncWriteStream>::Write(const void* buffer, int size)
{
boost::system::error_code ec;
m_Socket.write_some(boost::asio::buffer(buffer, size), ec);
return !ec;
}
Usage:
AsioInputStream<boost::asio::ip::tcp::socket> ais(m_Socket); // Where m_Socket is a instance of boost::asio::ip::tcp::socket
CopyingInputStreamAdaptor cis_adp(&ais);
CodedInputStream cis(&cis_adp);
Message protoMessage;
uint32_t msg_size;
/* Read message size */
if(!cis.ReadVarint32(&msg_size)) {
// Handle error
}
/* Make sure not to read beyond limit of message */
CodedInputStream::Limit msg_limit = cis.PushLimit(msg_size);
if(!msg.ParseFromCodedStream(&cis)) {
// Handle error
}
/* Remove limit */
cis.PopLimit(msg_limit);
Here you go:
#include <google/protobuf/io/zero_copy_stream_impl.h>
#include <google/protobuf/io/coded_stream.h>
using namespace google::protobuf::io;
class FASWriter
{
std::ofstream mFs;
OstreamOutputStream *_OstreamOutputStream;
CodedOutputStream *_CodedOutputStream;
public:
FASWriter(const std::string &file) : mFs(file,std::ios::out | std::ios::binary)
{
assert(mFs.good());
_OstreamOutputStream = new OstreamOutputStream(&mFs);
_CodedOutputStream = new CodedOutputStream(_OstreamOutputStream);
}
inline void operator()(const ::google::protobuf::Message &msg)
{
_CodedOutputStream->WriteVarint32(msg.ByteSize());
if ( !msg.SerializeToCodedStream(_CodedOutputStream) )
std::cout << "SerializeToCodedStream error " << std::endl;
}
~FASWriter()
{
delete _CodedOutputStream;
delete _OstreamOutputStream;
mFs.close();
}
};
class FASReader
{
std::ifstream mFs;
IstreamInputStream *_IstreamInputStream;
CodedInputStream *_CodedInputStream;
public:
FASReader(const std::string &file), mFs(file,std::ios::in | std::ios::binary)
{
assert(mFs.good());
_IstreamInputStream = new IstreamInputStream(&mFs);
_CodedInputStream = new CodedInputStream(_IstreamInputStream);
}
template<class T>
bool ReadNext()
{
T msg;
unsigned __int32 size;
bool ret;
if ( ret = _CodedInputStream->ReadVarint32(&size) )
{
CodedInputStream::Limit msgLimit = _CodedInputStream->PushLimit(size);
if ( ret = msg.ParseFromCodedStream(_CodedInputStream) )
{
_CodedInputStream->PopLimit(msgLimit);
std::cout << mFeed << " FASReader ReadNext: " << msg.DebugString() << std::endl;
}
}
return ret;
}
~FASReader()
{
delete _CodedInputStream;
delete _IstreamInputStream;
mFs.close();
}
};
I ran into the same issue in both C++ and Python.
For the C++ version, I used a mix of the code Kenton Varda posted on this thread and the code from the pull request he sent to the protobuf team (because the version posted here doesn't handle EOF while the one he sent to github does).
#include <google/protobuf/message_lite.h>
#include <google/protobuf/io/zero_copy_stream.h>
#include <google/protobuf/io/coded_stream.h>
bool writeDelimitedTo(const google::protobuf::MessageLite& message,
google::protobuf::io::ZeroCopyOutputStream* rawOutput)
{
// We create a new coded stream for each message. Don't worry, this is fast.
google::protobuf::io::CodedOutputStream output(rawOutput);
// Write the size.
const int size = message.ByteSize();
output.WriteVarint32(size);
uint8_t* buffer = output.GetDirectBufferForNBytesAndAdvance(size);
if (buffer != NULL)
{
// Optimization: The message fits in one buffer, so use the faster
// direct-to-array serialization path.
message.SerializeWithCachedSizesToArray(buffer);
}
else
{
// Slightly-slower path when the message is multiple buffers.
message.SerializeWithCachedSizes(&output);
if (output.HadError())
return false;
}
return true;
}
bool readDelimitedFrom(google::protobuf::io::ZeroCopyInputStream* rawInput, google::protobuf::MessageLite* message, bool* clean_eof)
{
// We create a new coded stream for each message. Don't worry, this is fast,
// and it makes sure the 64MB total size limit is imposed per-message rather
// than on the whole stream. (See the CodedInputStream interface for more
// info on this limit.)
google::protobuf::io::CodedInputStream input(rawInput);
const int start = input.CurrentPosition();
if (clean_eof)
*clean_eof = false;
// Read the size.
uint32_t size;
if (!input.ReadVarint32(&size))
{
if (clean_eof)
*clean_eof = input.CurrentPosition() == start;
return false;
}
// Tell the stream not to read beyond that size.
google::protobuf::io::CodedInputStream::Limit limit = input.PushLimit(size);
// Parse the message.
if (!message->MergeFromCodedStream(&input)) return false;
if (!input.ConsumedEntireMessage()) return false;
// Release the limit.
input.PopLimit(limit);
return true;
}
And here is my python2 implementation:
from google.protobuf.internal import encoder
from google.protobuf.internal import decoder
#I had to implement this because the tools in google.protobuf.internal.decoder
#read from a buffer, not from a file-like objcet
def readRawVarint32(stream):
mask = 0x80 # (1 << 7)
raw_varint32 = []
while 1:
b = stream.read(1)
#eof
if b == "":
break
raw_varint32.append(b)
if not (ord(b) & mask):
#we found a byte starting with a 0, which means it's the last byte of this varint
break
return raw_varint32
def writeDelimitedTo(message, stream):
message_str = message.SerializeToString()
delimiter = encoder._VarintBytes(len(message_str))
stream.write(delimiter + message_str)
def readDelimitedFrom(MessageType, stream):
raw_varint32 = readRawVarint32(stream)
message = None
if raw_varint32:
size, _ = decoder._DecodeVarint32(raw_varint32, 0)
data = stream.read(size)
if len(data) < size:
raise Exception("Unexpected end of file")
message = MessageType()
message.ParseFromString(data)
return message
#In place version that takes an already built protobuf object
#In my tests, this is around 20% faster than the other version
#of readDelimitedFrom()
def readDelimitedFrom_inplace(message, stream):
raw_varint32 = readRawVarint32(stream)
if raw_varint32:
size, _ = decoder._DecodeVarint32(raw_varint32, 0)
data = stream.read(size)
if len(data) < size:
raise Exception("Unexpected end of file")
message.ParseFromString(data)
return message
else:
return None
It might not be the best looking code and I'm sure it can be refactored a fair bit, but at least that should show you one way to do it.
Now the big problem: It's SLOW.
Even when using the C++ implementation of python-protobuf, it's one order of magnitude slower than in pure C++. I have a benchmark where I read 10M protobuf messages of ~30 bytes each from a file. It takes ~0.9s in C++, and 35s in python.
One way to make it a bit faster would be to re-implement the varint decoder to make it read from a file and decode in one go, instead of reading from a file and then decoding as this code currently does. (profiling shows that a significant amount of time is spent in the varint encoder/decoder). But needless to say that alone is not enough to close the gap between the python version and the C++ version.
Any idea to make it faster is very welcome :)
Just for completeness, I post here an up-to-date version that works with the master version of protobuf and Python3
For the C++ version it is sufficient to use the utils in delimited_message_utils.h, here a MWE
#include <google/protobuf/io/zero_copy_stream_impl.h>
#include <google/protobuf/util/delimited_message_util.h>
#include <stdio.h>
#include <fcntl.h>
#include <unistd.h>
template <typename T>
bool writeManyToFile(std::deque<T> messages, std::string filename) {
int outfd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC);
google::protobuf::io::FileOutputStream fout(outfd);
bool success;
for (auto msg: messages) {
success = google::protobuf::util::SerializeDelimitedToZeroCopyStream(
msg, &fout);
if (! success) {
std::cout << "Writing Failed" << std::endl;
break;
}
}
fout.Close();
close(outfd);
return success;
}
template <typename T>
std::deque<T> readManyFromFile(std::string filename) {
int infd = open(filename.c_str(), O_RDONLY);
google::protobuf::io::FileInputStream fin(infd);
bool keep = true;
bool clean_eof = true;
std::deque<T> out;
while (keep) {
T msg;
keep = google::protobuf::util::ParseDelimitedFromZeroCopyStream(
&msg, &fin, nullptr);
if (keep)
out.push_back(msg);
}
fin.Close();
close(infd);
return out;
}
For the Python3 version, building on #fireboot 's answer, the only thing thing that needed modification is the decoding of raw_varint32
def getSize(raw_varint32):
result = 0
shift = 0
b = six.indexbytes(raw_varint32, 0)
result |= ((ord(b) & 0x7f) << shift)
return result
def readDelimitedFrom(MessageType, stream):
raw_varint32 = readRawVarint32(stream)
message = None
if raw_varint32:
size = getSize(raw_varint32)
data = stream.read(size)
if len(data) < size:
raise Exception("Unexpected end of file")
message = MessageType()
message.ParseFromString(data)
return message
Was also looking for a solution for this. Here's the core of our solution, assuming some java code wrote many MyRecord messages with writeDelimitedTo into a file. Open the file and loop, doing:
if(someCodedInputStream->ReadVarint32(&bytes)) {
CodedInputStream::Limit msgLimit = someCodedInputStream->PushLimit(bytes);
if(myRecord->ParseFromCodedStream(someCodedInputStream)) {
//do your stuff with the parsed MyRecord instance
} else {
//handle parse error
}
someCodedInputStream->PopLimit(msgLimit);
} else {
//maybe end of file
}
Hope it helps.
Working with an objective-c version of protocol-buffers, I ran into this exact issue. On sending from the iOS client to a Java based server that uses parseDelimitedFrom, which expects the length as the first byte, I needed to call writeRawByte to the CodedOutputStream first. Posting here to hopegully help others that run into this issue. While working through this issue, one would think that Google proto-bufs would come with a simply flag which does this for you...
Request* request = [rBuild build];
[self sendMessage:request];
}
- (void) sendMessage:(Request *) request {
//** get length
NSData* n = [request data];
uint8_t len = [n length];
PBCodedOutputStream* os = [PBCodedOutputStream streamWithOutputStream:outputStream];
//** prepend it to message, such that Request.parseDelimitedFrom(in) can parse it properly
[os writeRawByte:len];
[request writeToCodedOutputStream:os];
[os flush];
}
Since I'm not allowed to write this as a comment to Kenton Varda's answer above; I believe there is a bug in the code he posted (as well as in other answers which have been provided). The following code:
...
google::protobuf::io::CodedInputStream input(rawInput);
// Read the size.
uint32_t size;
if (!input.ReadVarint32(&size)) return false;
// Tell the stream not to read beyond that size.
google::protobuf::io::CodedInputStream::Limit limit =
input.PushLimit(size);
...
sets an incorrect limit because it does not take into account the size of the varint32 which has already been read from input. This can result in data loss/corruption as additional bytes are read from the stream which may be part of the next message. The usual way of handling this correctly is to delete the CodedInputStream used to read the size and create a new one for reading the payload:
...
uint32_t size;
{
google::protobuf::io::CodedInputStream input(rawInput);
// Read the size.
if (!input.ReadVarint32(&size)) return false;
}
google::protobuf::io::CodedInputStream input(rawInput);
// Tell the stream not to read beyond that size.
google::protobuf::io::CodedInputStream::Limit limit =
input.PushLimit(size);
...
You can use getline for reading a string from a stream, using the specified delimiter:
istream& getline ( istream& is, string& str, char delim );
(defined in the header)