Develop Reference

Trying to do resume upload for AWS S3 with java sdk. But it is starting from scratch always to upload

I am trying to upload a file with AWS SDK to demonstrate upload break due to any reason. But I see it is uploading from the beginning always. The code there is pretty straight-forward, but the problem comes with saving persistent obj status so that we can re-use in resuming upload.
Here is my code:
public class Upload2 {
private static final File RESUME_UPLOAD_INFO = new File(System.getProperty("user.home"), "resumeUploadFile");
private static final boolean UPLOAD_HALF_AND_KILL = !RESUME_UPLOAD_INFO.exists();
private static Upload upload;
public static void main(String[] args) throws Exception
{
log("Execution started");
String bucket = "itpc123412";
String key = "s3.txt";
String file_path = "C:\\Users\\rkompelli\\Desktop\\abc3.txt";
File f = new File(file_path);
TransferManager tm = AWSConfiguration.getConnectionTransfer();
System.out.println(RESUME_UPLOAD_INFO.exists());
if (UPLOAD_HALF_AND_KILL)
{
PutObjectRequest por = new PutObjectRequest(bucket, key,f);
log("Starting from scratch");
upload = tm.upload(por, LISTENER);
}
else
{
log("Starting again");
upload = tm.resumeUpload(PersistableUpload.deserializeFrom(new FileInputStream(RESUME_UPLOAD_INFO)));
upload.addProgressListener(LISTENER);
}
log("Total bytes to transfer: " + upload.getProgress().getTotalBytesToTransfer());
log("Bytes transferred: " + upload.getProgress().getBytesTransferred());
upload.waitForCompletion();
log("Done");
tm.shutdownNow();
if (upload.isDone())
RESUME_UPLOAD_INFO.delete();
System.exit(0);
}
private static void log(String msg)
{
System.out.println(DateFormat.getTimeInstance().format(new Date()) + ": " + msg);
}
private static long LAST_LOGGED_BYTES;
private static final S3ProgressListener LISTENER = new S3SyncProgressListener()
{
#Override
public void progressChanged(ProgressEvent pe)
{
long bytes = upload.getProgress().getBytesTransferred();
if (pe.getEventType().isByteCountEvent())
{
if (LAST_LOGGED_BYTES == 0 || bytes - LAST_LOGGED_BYTES > 5)
{
LAST_LOGGED_BYTES = bytes;
log("uploaded: " + NumberFormat.getNumberInstance().format(bytes));
}
}
if (bytes > upload.getProgress().getTotalBytesToTransfer()/2 && UPLOAD_HALF_AND_KILL)
{
log("IF 2 in get progress");
log("Exiting");
System.exit(0);
}
}
#Override
public void onPersistableTransfer(final PersistableTransfer pt)
{
// TODO should not be blocked
try
{
log("Saving upload state");
pt.serialize(new FileOutputStream(RESUME_UPLOAD_INFO));
}
catch (Exception e)
{
e.printStackTrace();
}
}
};
}
I noticed that onPersistableTransfer() event is not executing which has code to serialize the obj.

Android Amazon S3 Upload. error 405

I have been trying for several days to connect Amazon S3 to my Android project. I downloaded the example "https://github.com/awslabs/aws-sdk-android-samples" "S3TransferUtilitySample" and everything works fine on it, I see the files through the aws admin panel.
I copied into my project "Constants.java" with the working settings, also copied "Util.java" without changes.
The purpose of my project is to record the file from the microphone and transfer it to the cloud.
Here is the singleton that should implement this operations :
public class RecorderHelper {
private static final String TAG = "UploadActivity";
private static TransferUtility sTransferUtility;
static private Util util;
static RecorderHelper singleton;
static Boolean RecordStateRecording;
private static MediaRecorder recorder;
private final String RECORD = Environment.getExternalStorageDirectory() + "/record.aac";
String fileName;
private RecorderHelper() {
}
public static RecorderHelper getSingleton(Context context) {
if (singleton == null) {
RecordStateRecording = false;
singleton = new RecorderHelper();
util = new Util();
AmazonS3Client s3Client = util.getS3Client(context);
sTransferUtility = util.getTransferUtility(context);
}
;
return singleton;
}
public void StopRecording() {
try {
if (RecordStateRecording) {
recorder.stop();
recorder.reset();
recorder.release();
AACTrackImpl aacTrack = new AACTrackImpl(new FileDataSourceImpl(RECORD));
if (aacTrack.getSamples().size() > 1000) {
CroppedTrack aacTrackShort = new CroppedTrack(aacTrack, aacTrack.getSamples().size() - 1000, aacTrack.getSamples().size());
Movie movie = new Movie();
movie.addTrack(aacTrackShort);
Container mp4file = new DefaultMp4Builder().build(movie);
FileChannel fc = new FileOutputStream(new File(fileName)).getChannel();
mp4file.writeContainer(fc);
fc.close();
aacTrackShort.close();
aacTrack.close();
} else {
aacTrack.close();
}
}
File file = new File(RECORD);
TransferObserver observer = sTransferUtility.upload(Constants.BUCKET_NAME, file.getName(),
file);
observer.setTransferListener(new UploadListener());
} catch (Exception e) {
Log.e("RECORD", e.getMessage());
}
RecordStateRecording = false;
}
public void StartNewRecording(String UUID) {
StopRecording();
recorder = new MediaRecorder();
recorder.setAudioSource(MediaRecorder.AudioSource.MIC);
recorder.setOutputFormat(MediaRecorder.OutputFormat.AAC_ADTS);
recorder.setAudioEncoder(MediaRecorder.AudioEncoder.AAC);
fileName = Environment.getExternalStorageDirectory() + "/" + UUID + ".aac";
recorder.setOutputFile(RECORD);
try {
recorder.prepare();
} catch (IOException e) {
e.printStackTrace();
}
recorder.start(); // Recording is now started
RecordStateRecording = true;
}
private class UploadListener implements TransferListener {
// Simply updates the UI list when notified.
#Override
public void onError(int id, Exception e) {
Log.e(TAG, "Error during upload: " + id, e);
}
#Override
public void onProgressChanged(int id, long bytesCurrent, long bytesTotal) {
Log.d(TAG, String.format("onProgressChanged: %d, total: %d, current: %d",
id, bytesTotal, bytesCurrent));
}
#Override
public void onStateChanged(int id, TransferState newState) {
Log.d(TAG, "onStateChanged: " + id + ", " + newState);
}
}
}
However, the file does not appear in the cloud and the listener tells me about the 405 error. Here is the full text.
Can anyone tell me what I'm doing wrong?
Unable to unmarshall error response (Attempt to invoke virtual method
'boolean java.lang.String.equals(java.lang.Object)' on a null object
reference). Response Code: 405, Response Text:
I'm ussing the latest SDK :
compile 'com.amazonaws:aws-android-sdk-s3:2.6.+'
Not sure about stacktrace because because i just get a callback to my listener about the transfer fails.
API 22

Using ScheduledExecutorService to save(Entites), I get detached Entity passed to persist error

I have a very curious Error that I cant seem to get my head around.
I need to use a ScheduledExecutorService to pass the Survey Entity I created to be edited and then saved as a new Entity.
public void executeScheduled(Survey eventObject, long interval) {
HashMap<String, String> eventRRules = StringUtils.extractSerialDetails(eventObject.getvCalendarRRule());
long delay = 10000;
ScheduledExecutorService service = Executors.newScheduledThreadPool(1);
Runnable runnable = new Runnable() {
private int counter = 1;
private int executions = Integer.parseInt(eventRRules.get("COUNT"));
Survey survey = eventObject;
public void run() {
String uid = eventObject.getUniqueEventId();
logger.info("SurveyController - executeScheduled - Iteration: " + counter);
String serialUid = null;
if (counter == 1) {
serialUid = uid + "-" + counter;
} else {
serialUid = StringUtils.removeLastAndConcatVar(eventObject.getUniqueEventId(), Integer.toString(counter));
}
if (++counter > executions) {
service.shutdown();
}
survey.setUniqueEventId(serialUid);
try {
executeCreateSurvey(survey);
} catch(Exception e) {
logger.debug("SurveyController - executeScheduled - Exception caught: ");
e.printStackTrace();
}
}
};
service.scheduleAtFixedRate(runnable, delay, interval, TimeUnit.MILLISECONDS);
}
When the executeCreateSurvey(survey) Method is run without the ScheduleExecutorService, it works flawlessly.
Yet when it is executed inside the run() Method, I get the "detached entity passed to persist" Error each time the save(survey) Method is run within the executeCreateSurvey() Method....
The executeCreateSurvey() Method where the .save() Method is called:
public ResponseEntity<?> executeCreateSurvey(Survey eventObject) {
MailService mailService = new MailService(applicationProperties);
Participant eventOwner = participantRepositoryImpl.createOrFindParticipant(eventObject.getEventOwner());
eventObject.setEventOwner(eventOwner);
Survey survey = surveyRepositoryImpl.createSurveyOrFindSurvey((Survey) eventObject);
// Saves additional information if small errors (content
// errors,.. )
// occurs
String warnMessage = "";
List<Participant> participants = new ArrayList<Participant>();
for (Participant participantDetached : eventObject.getParticipants()) {
// Check if participant already exists
Participant participant = participantRepositoryImpl.createOrFindParticipant(participantDetached);
participants.add(participant);
// Only create PartSur if not existing (Update case)
if (partSurRepository.findAllByParticipantAndSurvey(participant, survey).isEmpty()) {
PartSur partSur = new PartSur(participant, survey);
partSurRepository.save(partSur);
try {
mailService.sendRatingInvitationEmail(partSur);
surveyRepository.save(survey);
} catch (Exception e) {
// no special exception for "security" reasons
String errorMessage = "error sending mail for participant: " + e.getMessage() + "\n";
warnMessage += errorMessage;
logger.warn("createSurvey() - " + errorMessage);
}
}
}
// Delete all PartSurs and Answers from removed participants
List<PartSur> partSursForParticipantsRemoved = partSurRepository.findAllByParticipantNotIn(participants);
logger.warn("createSurvey() - participants removed: " + partSursForParticipantsRemoved.size());
partSurRepositoryImpl.invalidatePartSurs(partSursForParticipantsRemoved);
return new ResponseEntity<>("Umfrage wurde angelegt. Warnungen: " + warnMessage, HttpStatus.OK);
}
What could the reason be for this?
I have not been able to find this Problem anywhere so far.

How to add second activity in Amazon SWF hello_sample example

I've successfully implemented the simple Java Amazon SWF example called hello_sample. I created the ActivityWorker executable that polls SWF for activity tasks to process. I created the WorkflowWorker executable that polls SWF for decision tasks and I have a WorkflowStarter executable that kicks off the workflow execution. It works as advertised. What I don't understand is how do I configure and add a second activity to run after the first activity?
WorkflowWorker:
public class WorkflowWorker {
private static final AmazonSimpleWorkflow swf = AmazonSimpleWorkflowClientBuilder.defaultClient();
public static void main(String[] args) {
PollForDecisionTaskRequest task_request =
new PollForDecisionTaskRequest()
.withDomain(Constants.DOMAIN)
.withTaskList(new TaskList().withName(Constants.TASKLIST));
while (true) {
System.out.println(
"WorkflowWorker is polling for a decision task from the tasklist '" +
Constants.TASKLIST + "' in the domain '" +
Constants.DOMAIN + "'.");
DecisionTask task = swf.pollForDecisionTask(task_request);
String taskToken = task.getTaskToken();
if (taskToken != null) {
try {
executeDecisionTask(taskToken, task.getEvents());
}
catch (Throwable th) {
th.printStackTrace();
}
}
}
}
private static void executeDecisionTask(String taskToken, List<HistoryEvent> events) throws Throwable {
List<Decision> decisions = new ArrayList<Decision>();
String workflow_input = null;
int scheduled_activities = 0;
int open_activities = 0;
boolean activity_completed = false;
String result = null;
System.out.println("WorkflowWorker is executing the decision task for the history events: [");
for (HistoryEvent event : events) {
System.out.println(" " + event);
switch(event.getEventType()) {
case "WorkflowExecutionStarted":
workflow_input = event.getWorkflowExecutionStartedEventAttributes().getInput();
break;
case "ActivityTaskScheduled":
scheduled_activities++;
break;
case "ScheduleActivityTaskFailed":
scheduled_activities--;
break;
case "ActivityTaskStarted":
scheduled_activities--;
open_activities++;
break;
case "ActivityTaskCompleted":
open_activities--;
activity_completed = true;
result = event.getActivityTaskCompletedEventAttributes().getResult();
break;
case "ActivityTaskFailed":
open_activities--;
break;
case "ActivityTaskTimedOut":
open_activities--;
break;
}
}
System.out.println("]");
if (activity_completed) {
decisions.add(
new Decision()
.withDecisionType(DecisionType.CompleteWorkflowExecution)
.withCompleteWorkflowExecutionDecisionAttributes(
new CompleteWorkflowExecutionDecisionAttributes()
.withResult(result)));
}
else {
if (open_activities == 0 && scheduled_activities == 0) {
ScheduleActivityTaskDecisionAttributes attrs =
new ScheduleActivityTaskDecisionAttributes()
.withActivityType(new ActivityType()
.withName(Constants.ACTIVITY)
.withVersion(Constants.ACTIVITY_VERSION))
.withActivityId(UUID.randomUUID().toString())
.withInput(workflow_input);
decisions.add(
new Decision()
.withDecisionType(DecisionType.ScheduleActivityTask)
.withScheduleActivityTaskDecisionAttributes(attrs));
}
else {
// an instance of HelloActivity is already scheduled or running. Do nothing, another
// task will be scheduled once the activity completes, fails or times out
}
}
System.out.println("WorkflowWorker is exiting the decision task with the decisions " + decisions);
swf.respondDecisionTaskCompleted(
new RespondDecisionTaskCompletedRequest()
.withTaskToken(taskToken)
.withDecisions(decisions));
}
}
ActivityWorker:
public class ActivityWorker {
private static final AmazonSimpleWorkflow swf = AmazonSimpleWorkflowClientBuilder.defaultClient();
private static CountDownLatch waitForTermination = new CountDownLatch(1);
private static volatile boolean terminate = false;
private static String executeActivityTask(String g_species) throws Throwable {
String s = " ******** Hello, " + g_species + "!";
System.out.println(s);
String cwd = Paths.get(".").toAbsolutePath().normalize().toString();
String filename = "g_species.txt";
Path filePath = Paths.get(cwd, filename);
String filePathName = filePath.toString();
BufferedWriter output = null;
try {
File file = new File (filePathName);
output = new BufferedWriter(new FileWriter(file));
output.write(g_species);
}
catch (IOException e) {
e.printStackTrace();
}
finally {
if (output != null) {
output.close();
}
}
return g_species;
}
public static void main(String[] args) {
Runtime.getRuntime().addShutdownHook(new Thread() {
#Override
public void run() {
try {
terminate = true;
System.out.println("ActivityWorker is waiting for the current poll request to return before shutting down.");
waitForTermination.await(60, TimeUnit.SECONDS);
}
catch (InterruptedException e) {
// ignore
System.out.println(e.getMessage());
}
}
});
try {
pollAndExecute();
}
finally {
waitForTermination.countDown();
}
}
public static void pollAndExecute() {
while (!terminate) {
System.out.println("ActivityWorker is polling for an activity task from the tasklist '"
+ Constants.TASKLIST + "' in the domain '" + Constants.DOMAIN + "'.");
ActivityTask task = swf.pollForActivityTask(new PollForActivityTaskRequest()
.withDomain(Constants.DOMAIN)
.withTaskList(new TaskList().withName(Constants.TASKLIST)));
String taskToken = task.getTaskToken();
if (taskToken != null) {
String result = null;
Throwable error = null;
try {
System.out.println("ActivityWorker is executing the activity task with input '" + task.getInput() + "'.");
result = executeActivityTask(task.getInput());
}
catch (Throwable th) {
error = th;
}
if (error == null) {
System.out.println("The activity task succeeded with result '" + result + "'.");
swf.respondActivityTaskCompleted(
new RespondActivityTaskCompletedRequest()
.withTaskToken(taskToken)
.withResult(result));
}
else {
System.out.println("The activity task failed with the error '"
+ error.getClass().getSimpleName() + "'.");
swf.respondActivityTaskFailed(
new RespondActivityTaskFailedRequest()
.withTaskToken(taskToken)
.withReason(error.getClass().getSimpleName())
.withDetails(error.getMessage()));
}
}
}
}
}
WorkflowStarter that kicks it all off:
public class WorkflowStarter {
private static final AmazonSimpleWorkflow swf = AmazonSimpleWorkflowClientBuilder.defaultClient();
public static final String WORKFLOW_EXECUTION = "HelloWorldWorkflowExecution";
public static void main(String[] args) {
String workflow_input = "Amazon SWF";
if (args.length > 0) {
workflow_input = args[0];
}
System.out.println("Starting the workflow execution '" + WORKFLOW_EXECUTION +
"' with input '" + workflow_input + "'.");
WorkflowType wf_type = new WorkflowType()
.withName(Constants.WORKFLOW)
.withVersion(Constants.WORKFLOW_VERSION);
Run run = swf.startWorkflowExecution(new StartWorkflowExecutionRequest()
.withDomain(Constants.DOMAIN)
.withWorkflowType(wf_type)
.withWorkflowId(WORKFLOW_EXECUTION)
.withInput(workflow_input)
.withExecutionStartToCloseTimeout("90"));
System.out.println("Workflow execution started with the run id '" +
run.getRunId() + "'.");
}
}

I would recommend to not reinvent the wheel and use the AWS Flow Framework for Java that is officially supported by Amazon. It already implements all the low level details and allows you to focus on a business logic of your workflow directly.
Here is an example worklow that uses three activities (taken from the developer guide).
Activities interface:
import com.amazonaws.services.simpleworkflow.flow.annotations.Activities;
import com.amazonaws.services.simpleworkflow.flow.annotations.ActivityRegistrationOptions;
#ActivityRegistrationOptions(defaultTaskScheduleToStartTimeoutSeconds = 300,
defaultTaskStartToCloseTimeoutSeconds = 10)
#Activities(version="1.0")
public interface GreeterActivities {
public String getName();
public String getGreeting(String name);
public void say(String what);
}
Activities implementation:
public class GreeterActivitiesImpl implements GreeterActivities {
#Override
public String getName() {
return "World";
}
#Override
public String getGreeting(String name) {
return "Hello " + name;
}
#Override
public void say(String what) {
System.out.println(what);
}
}
Workflow interface:
import com.amazonaws.services.simpleworkflow.flow.annotations.Execute;
import com.amazonaws.services.simpleworkflow.flow.annotations.Workflow;
import com.amazonaws.services.simpleworkflow.flow.annotations.WorkflowRegistrationOptions;
#Workflow
#WorkflowRegistrationOptions(defaultExecutionStartToCloseTimeoutSeconds = 3600)
public interface GreeterWorkflow {
#Execute(version = "1.0")
public void greet();
}
Workflow implementation:
import com.amazonaws.services.simpleworkflow.flow.core.Promise;
public class GreeterWorkflowImpl implements GreeterWorkflow {
private GreeterActivitiesClient operations = new GreeterActivitiesClientImpl();
public void greet() {
Promise<String> name = operations.getName();
Promise<String> greeting = operations.getGreeting(name);
operations.say(greeting);
}
}
The worker that hosts both of them. Obviously it can be broken into separate activity and workflow workers:
import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.simpleworkflow.AmazonSimpleWorkflow;
import com.amazonaws.services.simpleworkflow.AmazonSimpleWorkflowClient;
import com.amazonaws.services.simpleworkflow.flow.ActivityWorker;
import com.amazonaws.services.simpleworkflow.flow.WorkflowWorker;
public class GreeterWorker {
public static void main(String[] args) throws Exception {
ClientConfiguration config = new ClientConfiguration().withSocketTimeout(70*1000);
String swfAccessId = System.getenv("AWS_ACCESS_KEY_ID");
String swfSecretKey = System.getenv("AWS_SECRET_KEY");
AWSCredentials awsCredentials = new BasicAWSCredentials(swfAccessId, swfSecretKey);
AmazonSimpleWorkflow service = new AmazonSimpleWorkflowClient(awsCredentials, config);
service.setEndpoint("https://swf.us-east-1.amazonaws.com");
String domain = "helloWorldWalkthrough";
String taskListToPoll = "HelloWorldList";
ActivityWorker aw = new ActivityWorker(service, domain, taskListToPoll);
aw.addActivitiesImplementation(new GreeterActivitiesImpl());
aw.start();
WorkflowWorker wfw = new WorkflowWorker(service, domain, taskListToPoll);
wfw.addWorkflowImplementationType(GreeterWorkflowImpl.class);
wfw.start();
}
}
The workflow starter:
import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.simpleworkflow.AmazonSimpleWorkflow;
import com.amazonaws.services.simpleworkflow.AmazonSimpleWorkflowClient;
public class GreeterMain {
public static void main(String[] args) throws Exception {
ClientConfiguration config = new ClientConfiguration().withSocketTimeout(70*1000);
String swfAccessId = System.getenv("AWS_ACCESS_KEY_ID");
String swfSecretKey = System.getenv("AWS_SECRET_KEY");
AWSCredentials awsCredentials = new BasicAWSCredentials(swfAccessId, swfSecretKey);
AmazonSimpleWorkflow service = new AmazonSimpleWorkflowClient(awsCredentials, config);
service.setEndpoint("https://swf.us-east-1.amazonaws.com");
String domain = "helloWorldWalkthrough";
GreeterWorkflowClientExternalFactory factory = new GreeterWorkflowClientExternalFactoryImpl(service, domain);
GreeterWorkflowClientExternal greeter = factory.getClient("someID");
greeter.greet();
}
}

Spymemcached set does not work when used with multithreading and high load

I am using memcached version 1.4.7 and spymemcached 2.8.4 as a client to set and get the key values to it. When used in multi-threaded and high load environment spymemcached client is unable to set the values in cache itself.
I am running my load test program with 40M long key which are equally divided in 20 worker threads. Each worker thread tries to set 1M keys in cache. Hence there are 40 worker threads running.
In my DefaultCache.java file, I have made a connection pool of 20 spymemcached clients. Every time a worker thread tries to set the key to cache DefaultCache.java returns it a random client as shown in getCache() method.
When my program exits, it prints
Total no of keys loaded = 40000000
However when I go to memcached telnet console, it always misses few thousands records. I have also verified it by randomly fetching few keys which output null. There is no eviction and the cmd_set, curr_items, total_items are each equal to 39.5M
What could be the reason behind these missing keys in cache.
Here is the code for reference purpose.
public class TestCacheLoader {
public static final Long TOTAL_RECORDS = 40000000L;
public static final Long LIMIT = 1000000L;
public static void main(String[] args) {
long keyCount = loadKeyCacheData();
System.out.println("Total no of keys loaded = " + keyCount);
}
public static long loadKeyCacheData() {
DefaultCache cache = new DefaultCache();
List<Future<Long>> futureList = new ArrayList<Future<Long>>();
ExecutorService executorThread = Executors.newFixedThreadPool(40);
long offset = 0;
long keyCount = 0;
long workerCount = 0;
try {
do {
List<Long> keyList = new ArrayList<Long>(LIMIT.intValue());
for (long counter = offset; counter < (offset + LIMIT) && counter < TOTAL_RECORDS; counter++) {
keyList.add(counter);
}
if (keyList.size() != 0) {
System.out.println("Initiating a new worker thread " + workerCount++);
KeyCacheThread keyCacheThread = new KeyCacheThread(keyList, cache);
futureList.add(executorThread.submit(keyCacheThread));
}
offset += LIMIT;
} while (offset < TOTAL_RECORDS);
for (Future<Long> future : futureList) {
keyCount += (Long) future.get();
}
} catch (Exception e) {
e.printStackTrace();
} finally {
cache.shutdown();
}
return keyCount;
}
}
class KeyCacheThread implements Callable<Long> {
private List<Long> keyList;
private DefaultCache cache;
public KeyCacheThread(List<Long> keyList, DefaultCache cache) {
this.keyList = keyList;
this.cache = cache;
}
public Long call() {
return createKeyCache();
}
public Long createKeyCache() {
String compoundKey = "";
long keyCounter = 0;
System.out.println(Thread.currentThread() + " started to process " + keyList.size() + " keys");
for (Long key : keyList) {
keyCounter++;
compoundKey = key.toString();
cache.set(compoundKey, 0, key);
}
System.out.println(Thread.currentThread() + " processed = " + keyCounter + " keys");
return keyCounter;
}
}
public class DefaultCache {
private static final Logger LOGGER = Logger.getLogger(DefaultCache.class);
private MemcachedClient[] clients;
public DefaultCache() {
this.cacheNamespace = "";
this.cacheName = "keyCache";
this.addresses = "127.0.0.1:11211";
this.cacheLookupTimeout = 3000;
this.numberOfClients = 20;
try {
LOGGER.debug("Cache initialization started for the cache : " + cacheName);
ConnectionFactory connectionFactory = new DefaultConnectionFactory(DefaultConnectionFactory.DEFAULT_OP_QUEUE_LEN,
DefaultConnectionFactory.DEFAULT_READ_BUFFER_SIZE, DefaultHashAlgorithm.KETAMA_HASH) {
public NodeLocator createLocator(List<MemcachedNode> list) {
KetamaNodeLocator locator = new KetamaNodeLocator(list, DefaultHashAlgorithm.KETAMA_HASH);
return locator;
}
};
clients = new MemcachedClient[numberOfClients];
for (int i = 0; i < numberOfClients; i++) {
MemcachedClient client = new MemcachedClient(connectionFactory, AddrUtil.getAddresses(getServerAddresses(addresses)));
clients[i] = client;
}
LOGGER.debug("Cache initialization ended for the cache : " + cacheName);
} catch (IOException e) {
LOGGER.error("Exception occured while initializing cache : " + cacheName, e);
throw new CacheException("Exception occured while initializing cache : " + cacheName, e);
}
}
public Object get(String key) {
try {
return getCache().get(cacheNamespace + key);
} catch (Exception e) {
return null;
}
}
public void set(String key, Integer expiryTime, final Object value) {
getCache().set(cacheNamespace + key, expiryTime, value);
}
public Object delete(String key) {
return getCache().delete(cacheNamespace + key);
}
public void shutdown() {
for (MemcachedClient client : clients) {
client.shutdown();
}
}
public void flush() {
for (MemcachedClient client : clients) {
client.flush();
}
}
private MemcachedClient getCache() {
MemcachedClient client = null;
int i = (int) (Math.random() * numberOfClients);
client = clients[i];
return client;
}
private String getServerAddresses(List<Address> addresses) {
StringBuilder addressStr = new StringBuilder();
for (Address address : addresses) {
addressStr.append(address.getHost()).append(":").append(address.getPort()).append(" ");
}
return addressStr.toString().trim();
}
}

I saw the same. The reason is reactor pattern that they use for async operations. That means 1 worker thread per 1 connection. This 1 thread is a bootleneck under high load and multi threaded machines. 1 thread can load only 1 CPU while rest 23 will be idle.
We have come up with pool of connections that increased the worker threads and allowed to utilize more hardware power. Check out project 3levelmemcache at github.

I am not sure but it seems the issue with spymemcached library itself.
I changed the implemention of DefaultCache.java file to use xmemcached and everything started working fine. Now I am not missing any records. telnet stats are showing matching number of set commands.
Thanks for your patience though.