This MainActivity.java was written for quantised models and I'm trying to use unquantised model.
After making the changes as mentioned here, here to MainActivity.java, my code is
public class MainActivity extends AppCompatActivity implements AdapterView.OnItemSelectedListener {
private static final String TAG = "MainActivity";
private Button mRun;
private ImageView mImageView;
private Bitmap mSelectedImage;
private GraphicOverlay mGraphicOverlay;
// Max width (portrait mode)
private Integer mImageMaxWidth;
// Max height (portrait mode)
private Integer mImageMaxHeight;
private final String[] mFilePaths =
new String[]{"mountain.jpg", "tennis.jpg","96580.jpg"};
/**
* Name of the model file hosted with Firebase.
*/
private static final String HOSTED_MODEL_NAME = "mobilenet_v1_224_quant";
private static final String LOCAL_MODEL_ASSET = "retrained_graph_mobilenet_1_224.tflite";
/**
* Name of the label file stored in Assets.
*/
private static final String LABEL_PATH = "labels.txt";
/**
* Number of results to show in the UI.
*/
private static final int RESULTS_TO_SHOW = 3;
/**
* Dimensions of inputs.
*/
private static final int DIM_BATCH_SIZE = 1;
private static final int DIM_PIXEL_SIZE = 3;
private static final int DIM_IMG_SIZE_X = 224;
private static final int DIM_IMG_SIZE_Y = 224;
private static final int IMAGE_MEAN = 128;
private static final float IMAGE_STD = 128.0f;
/**
* Labels corresponding to the output of the vision model.
*/
private List<String> mLabelList;
private final PriorityQueue<Map.Entry<String, Float>> sortedLabels =
new PriorityQueue<>(
RESULTS_TO_SHOW,
new Comparator<Map.Entry<String, Float>>() {
#Override
public int compare(Map.Entry<String, Float> o1, Map.Entry<String, Float>
o2) {
return (o1.getValue()).compareTo(o2.getValue());
}
});
/* Preallocated buffers for storing image data. */
private final int[] intValues = new int[DIM_IMG_SIZE_X * DIM_IMG_SIZE_Y];
/**
* An instance of the driver class to run model inference with Firebase.
*/
private FirebaseModelInterpreter mInterpreter;
/**
* Data configuration of input & output data of model.
*/
private FirebaseModelInputOutputOptions mDataOptions;
#Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
mGraphicOverlay = findViewById(R.id.graphic_overlay);
mImageView = findViewById(R.id.image_view);
Spinner dropdown = findViewById(R.id.spinner);
List<String> items = new ArrayList<>();
for (int i = 0; i < mFilePaths.length; i++) {
items.add("Image " + (i + 1));
}
ArrayAdapter<String> adapter = new ArrayAdapter<>(this, android.R.layout
.simple_spinner_dropdown_item, items);
dropdown.setAdapter(adapter);
dropdown.setOnItemSelectedListener(this);
mLabelList = loadLabelList(this);
mRun = findViewById(R.id.button_run);
mRun.setOnClickListener(new View.OnClickListener() {
#Override
public void onClick(View v) {
runModelInference();
}
});
int[] inputDims = {DIM_BATCH_SIZE, DIM_IMG_SIZE_X, DIM_IMG_SIZE_Y, DIM_PIXEL_SIZE};
int[] outputDims = {DIM_BATCH_SIZE, mLabelList.size()};
try {
mDataOptions =
new FirebaseModelInputOutputOptions.Builder()
.setInputFormat(0, FirebaseModelDataType.FLOAT32, inputDims)
.setOutputFormat(0, FirebaseModelDataType.FLOAT32, outputDims)
.build();
FirebaseModelDownloadConditions conditions = new FirebaseModelDownloadConditions
.Builder()
.requireWifi()
.build();
FirebaseLocalModelSource localModelSource =
new FirebaseLocalModelSource.Builder("asset")
.setAssetFilePath(LOCAL_MODEL_ASSET).build();
FirebaseCloudModelSource cloudSource = new FirebaseCloudModelSource.Builder
(HOSTED_MODEL_NAME)
.enableModelUpdates(true)
.setInitialDownloadConditions(conditions)
.setUpdatesDownloadConditions(conditions) // You could also specify
// different conditions
// for updates
.build();
FirebaseModelManager manager = FirebaseModelManager.getInstance();
manager.registerLocalModelSource(localModelSource);
manager.registerCloudModelSource(cloudSource);
FirebaseModelOptions modelOptions =
new FirebaseModelOptions.Builder()
.setCloudModelName(HOSTED_MODEL_NAME)
.setLocalModelName("asset")
.build();
mInterpreter = FirebaseModelInterpreter.getInstance(modelOptions);
} catch (FirebaseMLException e) {
showToast("Error while setting up the model");
e.printStackTrace();
}
}
private void runModelInference() {
if (mInterpreter == null) {
Log.e(TAG, "Image classifier has not been initialized; Skipped.");
return;
}
// Create input data.
ByteBuffer imgData = convertBitmapToByteBuffer(mSelectedImage, mSelectedImage.getWidth(),
mSelectedImage.getHeight());
try {
FirebaseModelInputs inputs = new FirebaseModelInputs.Builder().add(imgData).build();
// Here's where the magic happens!!
mInterpreter
.run(inputs, mDataOptions)
.addOnFailureListener(new OnFailureListener() {
#Override
public void onFailure(#NonNull Exception e) {
e.printStackTrace();
showToast("Error running model inference");
}
})
.continueWith(
new Continuation<FirebaseModelOutputs, List<String>>() {
#Override
public List<String> then(Task<FirebaseModelOutputs> task) {
float[][] labelProbArray = task.getResult()
.<float[][]>getOutput(0);
List<String> topLabels = getTopLabels(labelProbArray);
mGraphicOverlay.clear();
GraphicOverlay.Graphic labelGraphic = new LabelGraphic
(mGraphicOverlay, topLabels);
mGraphicOverlay.add(labelGraphic);
return topLabels;
}
});
} catch (FirebaseMLException e) {
e.printStackTrace();
showToast("Error running model inference");
}
}
/**
* Gets the top labels in the results.
*/
private synchronized List<String> getTopLabels(float[][] labelProbArray) {
for (int i = 0; i < mLabelList.size(); ++i) {
sortedLabels.add(
new AbstractMap.SimpleEntry<>(mLabelList.get(i), (labelProbArray[0][i] )));
if (sortedLabels.size() > RESULTS_TO_SHOW) {
sortedLabels.poll();
}
}
List<String> result = new ArrayList<>();
final int size = sortedLabels.size();
for (int i = 0; i < size; ++i) {
Map.Entry<String, Float> label = sortedLabels.poll();
result.add(label.getKey() + ":" + label.getValue());
}
Log.d(TAG, "labels: " + result.toString());
return result;
}
/**
* Reads label list from Assets.
*/
private List<String> loadLabelList(Activity activity) {
List<String> labelList = new ArrayList<>();
try (BufferedReader reader =
new BufferedReader(new InputStreamReader(activity.getAssets().open
(LABEL_PATH)))) {
String line;
while ((line = reader.readLine()) != null) {
labelList.add(line);
}
} catch (IOException e) {
Log.e(TAG, "Failed to read label list.", e);
}
return labelList;
}
/**
* Writes Image data into a {#code ByteBuffer}.
*/
private synchronized ByteBuffer convertBitmapToByteBuffer(
Bitmap bitmap, int width, int height) {
ByteBuffer imgData =
ByteBuffer.allocateDirect(
4*DIM_BATCH_SIZE * DIM_IMG_SIZE_X * DIM_IMG_SIZE_Y * DIM_PIXEL_SIZE);
imgData.order(ByteOrder.nativeOrder());
Bitmap scaledBitmap = Bitmap.createScaledBitmap(bitmap, DIM_IMG_SIZE_X, DIM_IMG_SIZE_Y,
true);
imgData.rewind();
scaledBitmap.getPixels(intValues, 0, scaledBitmap.getWidth(), 0, 0,
scaledBitmap.getWidth(), scaledBitmap.getHeight());
// Convert the image to int points.
int pixel = 0;
for (int i = 0; i < DIM_IMG_SIZE_X; ++i) {
for (int j = 0; j < DIM_IMG_SIZE_Y; ++j) {
final int val = intValues[pixel++];
imgData.putFloat((((val >> 16) & 0xFF)-IMAGE_MEAN)/IMAGE_STD);
imgData.putFloat((((val >> 8) & 0xFF)-IMAGE_MEAN)/IMAGE_STD);
imgData.putFloat(((val & 0xFF)-IMAGE_MEAN)/IMAGE_STD);
}
}
return imgData;
}
private void showToast(String message) {
Toast.makeText(getApplicationContext(), message, Toast.LENGTH_SHORT).show();
}
public void onItemSelected(AdapterView<?> parent, View v, int position, long id) {
mGraphicOverlay.clear();
mSelectedImage = getBitmapFromAsset(this, mFilePaths[position]);
if (mSelectedImage != null) {
// Get the dimensions of the View
Pair<Integer, Integer> targetedSize = getTargetedWidthHeight();
int targetWidth = targetedSize.first;
int maxHeight = targetedSize.second;
// Determine how much to scale down the image
float scaleFactor =
Math.max(
(float) mSelectedImage.getWidth() / (float) targetWidth,
(float) mSelectedImage.getHeight() / (float) maxHeight);
Bitmap resizedBitmap =
Bitmap.createScaledBitmap(
mSelectedImage,
(int) (mSelectedImage.getWidth() / scaleFactor),
(int) (mSelectedImage.getHeight() / scaleFactor),
true);
mImageView.setImageBitmap(resizedBitmap);
mSelectedImage = resizedBitmap;
}
}
#Override
public void onNothingSelected(AdapterView<?> parent) {
// Do nothing
}
// Utility functions for loading and resizing images from app asset folder.
public static Bitmap getBitmapFromAsset(Context context, String filePath) {
AssetManager assetManager = context.getAssets();
InputStream is;
Bitmap bitmap = null;
try {
is = assetManager.open(filePath);
bitmap = BitmapFactory.decodeStream(is);
} catch (IOException e) {
e.printStackTrace();
}
return bitmap;
}
// Returns max image width, always for portrait mode. Caller needs to swap width / height for
// landscape mode.
private Integer getImageMaxWidth() {
if (mImageMaxWidth == null) {
// Calculate the max width in portrait mode. This is done lazily since we need to
// wait for a UI layout pass to get the right values. So delay it to first time image
// rendering time.
mImageMaxWidth = mImageView.getWidth();
}
return mImageMaxWidth;
}
// Returns max image height, always for portrait mode. Caller needs to swap width / height for
// landscape mode.
private Integer getImageMaxHeight() {
if (mImageMaxHeight == null) {
// Calculate the max width in portrait mode. This is done lazily since we need to
// wait for a UI layout pass to get the right values. So delay it to first time image
// rendering time.
mImageMaxHeight =
mImageView.getHeight();
}
return mImageMaxHeight;
}
// Gets the targeted width / height.
private Pair<Integer, Integer> getTargetedWidthHeight() {
int targetWidth;
int targetHeight;
int maxWidthForPortraitMode = getImageMaxWidth();
int maxHeightForPortraitMode = getImageMaxHeight();
targetWidth = maxWidthForPortraitMode;
targetHeight = maxHeightForPortraitMode;
return new Pair<>(targetWidth, targetHeight);
}
}
But I'm still getting Failed to get input dimensions. 0-th input should have 268203 bytes, but found 1072812 bytes for inception and 0-th input should have 150528 bytes, but found 602112 bytes for mobilenet. So, a factor is 4 there always.
To see what I've changed, the output of diff original.java changed.java is: (Ignore the line numbers)
32a33,34
> private static final int IMAGE_MEAN = 128;
> private static final float IMAGE_STD = 128.0f;
150,151c152,153
< byte[][] labelProbArray = task.getResult()
< .<byte[][]>getOutput(0);
---
> float[][] labelProbArray = task.getResult()
> .<float[][]>getOutput(0);
170c172
< private synchronized List<String> getTopLabels(byte[][] labelProbArray) {
---
> private synchronized List<String> getTopLabels(float[][] labelProbArray) {
173,174c175
< new AbstractMap.SimpleEntry<>(mLabelList.get(i), (labelProbArray[0][i] &
< 0xff) / 255.0f));
---
> new AbstractMap.SimpleEntry<>(mLabelList.get(i), (labelProbArray[0][i] )));
214c215,216
< DIM_BATCH_SIZE * DIM_IMG_SIZE_X * DIM_IMG_SIZE_Y * DIM_PIXEL_SIZE);
---
> 4*DIM_BATCH_SIZE * DIM_IMG_SIZE_X * DIM_IMG_SIZE_Y * DIM_PIXEL_SIZE);
>
226,228c228,232
< imgData.put((byte) ((val >> 16) & 0xFF));
< imgData.put((byte) ((val >> 8) & 0xFF));
< imgData.put((byte) (val & 0xFF));
---
> imgData.putFloat((((val >> 16) & 0xFF)-IMAGE_MEAN)/IMAGE_STD);
> imgData.putFloat((((val >> 8) & 0xFF)-IMAGE_MEAN)/IMAGE_STD);
> imgData.putFloat(((val & 0xFF)-IMAGE_MEAN)/IMAGE_STD);
This is how the buffer is allocated in the code lab:
ByteBuffer imgData = ByteBuffer.allocateDirect(
DIM_BATCH_SIZE * DIM_IMG_SIZE_X * DIM_IMG_SIZE_Y * DIM_PIXEL_SIZE);
DIM_BATCH_SIZE - A typical usage is for supporting batch processing (if the model supports it). In our sample and probably your test, you feed 1 image at a time and just keep it as 1.
DIM_PIXEL_SIZE - We set 3 in the code lab, which corresponds to r/g/b 1 byte each.
However, looks like you are using a float model. Then instead of one byte each for r/g/b, you use a float (4 bytes) to represent r/g/b each (you figured out this part already yourself). Then the buffer you allocated using above code is no longer sufficient.
You can follow example here for float models:
https://github.com/tensorflow/tensorflow/blob/25b4086bb5ba1788ceb6032eda58348f6e20a71d/tensorflow/contrib/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/ImageClassifierFloatInception.java
To be exact on imgData population, below should be the formula for allocation:
ByteBuffer imgData = ByteBuffer.allocateDirect(
DIM_BATCH_SIZE * getImageSizeX() * getImageSizeY() * DIM_PIXEL_SIZE
* getNumBytesPerChannel());
getNumBytesPerChannel() should be 4 in your case.
[Update for the new question, in regards of below error]:
Failed to get input dimensions. 0-th input should have 268203 bytes, but found 1072812 bytes
This is the check that number of bytes expected by the model == number of bytes passed in. 268203 = 299 * 299 * 3 & 1072812 = 4 * 299 * 299 * 3. Looks like you are using a quantized model but fed it with data for float model. Could you double check the model you used? To make things simple, don't specify cloud model source and use local model from assets only.
[Update 0628, developer said they trained a float model]:
It could be your model is wrong; it could also be you have a Cloud model downloaded which overrides your local model. But the error message tells us that the model being loaded is NOT a float model.
To isolate the issue, I'd recommend below few testings:
1) Remove setCloudModelName / registerCloudModelSource from quick start app
2) Play with official TFLite float model You will have to download the model mentioned in comment and change Camera2BasicFragment to use that ImageClassifierFloatInception (instead of ImageClassifierQuantizedMobileNet)
3) Still use the same TFLite sample app, switch to your own trained model. Make sure to tune the image size to your values.
Related
Hi I am using MPAndroid chart as charting library in my android app. I am trying to highlight some out of the range values on line dataset with different color (Picture 1)and if the value is beyond range then I want to also change marker-view drawable image also.
I have achieved this (Picture 2 ) for now I have managed to change color to red of out of the range values. How can I achieve Picture 1 in chart?
private void populateChart() {
chart = binding.lcCharts;
chart.setBackgroundColor(Color.WHITE);
chart.getDescription().setEnabled(false);
chart.setDoubleTapToZoomEnabled(false);
chart.setPinchZoom(false);
chart.setScaleEnabled(false);
getXAxisData()
LineData lineData = new LineData(setLineDataSet());
lineData.setDrawValues(false);
chart.setData(lineData);
chart.setTouchEnabled(true);
chart.setDrawMarkers(false);
chart.setHighlightPerTapEnabled(true);
chart.setMarker(new YourMarkerView(fragment.requireContext(), R.layout.layout_pop_up));
chart.setClipChildren(false);
chart.setClipToPadding(false);
chart.invalidate();
chart.notifyDataSetChanged();
}
private ArrayList<ILineDataSet> setLineDataSet() {
ArrayList<ILineDataSet> dataSet = new ArrayList<>();
for (int i = 0; i < response.size(); i++) {
LineDataSet lineDataSet = new LineDataSet(setData(i),
response.get(i).getName());
lineDataSet.setLineWidth(3);
lineDataSet.setColor(this.getResources().getColor(colorArray[i]));
lineDataSet.setDrawCircleHole(false);
lineDataSet.setCircleRadius(4);
lineDataSet.setCircleColors(setColorOfLineDataSet(i));
dataSet.add(lineDataSet);
}
return dataSet;
}
private ArrayList<Integer> setColorOfLineDataSet(int i) {
ArrayList<Integer> color = new ArrayList<Integer>();
for (int j = 0; j < response.get(i).size(); j++) {
if (!response.get(i).isNormal()) {
color.add(R.color.Red);
} else {
color.add(colorArray[i]);
}
}
return color;
} private void getXAxisData() {
XAxis xAxis = chart.getXAxis();
xAxis.setPosition(XAxis.XAxisPosition.BOTTOM);
xAxis.setDrawAxisLine(false);
xAxis.setDrawGridLines(false);
// xAxis.setGranularity(1f);
xAxis.setValueFormatter(new MyXAxisValueFormatter(getDateArrayForXAxis()));//getDateArrayForXAxis function returns a String array of size 14 with values of dates of past two weeks.
xAxis.setLabelCount(14, true);
}
public class MyXAxisValueFormatter extends ValueFormatter {
private String[] mValues;
public MyXAxisValueFormatter(String[] values) {
this.mValues = values;
}
public String getFormattedValue(float value) {
String val = null;
try {
val = String.valueOf(mValues[(int) value]);
} catch (IndexOutOfBoundsException e) {
}
return val;
}
/**
* Used to draw axis labels, calls {#link #getFormattedValue(float)} by default.
*
* #param value float to be formatted
* #param axis axis being labeled
* #return formatted string label
*/
public String getAxisLabel(float value, AxisBase axis) {
return getFormattedValue(value);
}
}
This code is crashing with arrayIndexOutOfBoundException.
enter code here
For customizing the Circle shape you can use following methods of the LineDataSet:
setCircleColor(/some Color value/)
setCircleRadius(/* some float value */)
setDrawCircleHole(/* boolean value */)
setDrawFilled(/* boolean value*/)
try these out, maybe you'll find something which you want. But I don't think it will allow you to change the shape completely.
I have referred to this article for an example:https://www.studytutorial.in/android-line-chart-or-line-graph-using-mpandroid-library-tutorial
example
What I've tried.
Please help me.. I really don't know.
I think it's related to canvas class.
CustomLineChartRenderer
It seems to me that color is not filled according to x value, but color is filled at once.
It's hard because I'm not used to canvas. Please help me.
Here are three methods for drawing lines and filling in colors.
#Override
protected void drawLinearFill(Canvas c, ILineDataSet dataSet, Transformer trans, XBounds bounds) {
final Path filled = mGenerateFilledPathBuffer;
final int startingIndex = bounds.min;
final int endingIndex = bounds.range + bounds.min;
final int indexInterval = 128;
int currentStartIndex = 0;
int currentEndIndex = indexInterval;
int iterations = 0;
// Doing this iteratively in order to avoid OutOfMemory errors that can happen on large bounds sets.
do {
currentStartIndex = startingIndex + (iterations * indexInterval);
currentEndIndex = currentStartIndex + indexInterval;
currentEndIndex = currentEndIndex > endingIndex ? endingIndex : currentEndIndex;
if (currentStartIndex <= currentEndIndex) {
generateFilledPath(dataSet, currentStartIndex, currentEndIndex, filled);
trans.pathValueToPixel(filled);
final Drawable drawable = dataSet.getFillDrawable();
if (drawable != null) {
drawFilledPath(c, filled, drawable);
} else {
//////Here part of applying color
drawFilledPath(c, filled, dataSet.getFillColor(), dataSet.getFillAlpha());
}
}
iterations++;
} while (currentStartIndex <= currentEndIndex);
}
#Override
protected void drawFilledPath(Canvas c, Path filledPath, int fillColor, int fillAlpha) {
int color = (fillAlpha << 24) | (fillColor & 0xffffff);
if (clipPathSupported()) {
Log.e("clipPathSupported","1");
int save = c.save();
c.clipPath(filledPath);
c.drawColor(color);
c.restoreToCount(save);
} else {
Log.e("clipPathSupported","2");
// save
Paint.Style previous = mRenderPaint.getStyle();
int previousColor = mRenderPaint.getColor();
// set
mRenderPaint.setStyle(Paint.Style.FILL);
mRenderPaint.setColor(color);
c.drawPath(filledPath, mRenderPaint);
// restore
mRenderPaint.setColor(previousColor);
mRenderPaint.setStyle(previous);
}
}
private void generateFilledPath(final ILineDataSet dataSet, final int startIndex, final int endIndex, final Path outputPath) {
final float fillMin = dataSet.getFillFormatter().getFillLinePosition(dataSet, mChart);
final float phaseY = mAnimator.getPhaseY();
final boolean isDrawSteppedEnabled = dataSet.getMode() == LineDataSet.Mode.STEPPED;
final Path filled = outputPath;
filled.reset();
final Entry entry = dataSet.getEntryForIndex(startIndex);
filled.moveTo(entry.getX(), fillMin);
filled.lineTo(entry.getX(), entry.getY() * phaseY);
// create a new path
Entry currentEntry = null;
Entry previousEntry = entry;
for (int x = startIndex + 1; x <= endIndex; x++) {
currentEntry = dataSet.getEntryForIndex(x);
if (isDrawSteppedEnabled) {
filled.lineTo(currentEntry.getX(), previousEntry.getY() * phaseY);
}
filled.lineTo(currentEntry.getX(), currentEntry.getY() * phaseY);
previousEntry = currentEntry;
}
// close up
if (currentEntry != null) {
filled.lineTo(currentEntry.getX(), fillMin);
}
filled.close();
}
You can use shader, same :
mRenderPaint.shader = LinearGradient(fromX, fromY, toX, toY, color1,
color2,
Shader.TileMode.CLAMP)
I'm interesting in creating a layered tif with Java in a way that Photoshop will recognize the layers. I was able to create a multi-page tif, but Photoshop does not recognize the pages as layers. The pages are viewable with Acrobat though. Anyone know how Photoshop stores tif layer data and how that could be generated with Java?
Thanks.
I have researched this for my TIFF ImageIO plugin, and as far as I understand, the way Photoshop stores layer information in TIFFs is completely proprietary and not using standard TIFF mechanisms, like multi-page documents utilizing linked or nested IFDs (330/SubIFD), or file types (254/NewSubFileType), etc.
Instead, it stores the layer information,
along with the layer image data, in a Photoshop specific TIFF tag; 37724/ImageSourceData, which has type UNDEFINED (or "just bytes"). Luckily, the contents of this tag is documented in Adobe Photoshop®
TIFF Technical Notes.
The content of this tag will always start with the 0-terminated string "Adobe Photoshop Document Data Block". The rest of the contents is various Photoshop resources, identified by the Photoshop 4 byte resource identifier 8BIM, followed 4 bytes resource key and 4 bytes length for each individual resource.
The interesting resource in this block, with regards to Photoshop layers, is the one identified with the resource key Layr. This is the same structure documented in Layer and Mask Information Section in the Photoshop File Format.
There's also a different tag, 34377/Photoshop, which contains other image resources read and written by Photoshop. It's also documented in the Image Resources Section of the above document. It does contain some information which is interesting in regards to layers, but I'm not sure how much of this you need to write. You will probably need a Photoshop installation and test using the "real thing".
I do have code to read both of these structures in the PSD ImageIO plugin, which might be worth looking at, but it doesn't yet support writing.
When you can write the contents Photoshop TIFF tags, you should be able to pass it to the TIFFImageWriter as part of the TIFF IIOMetadata and the writer will write it along with any other metadata and pixel data you pass.
So, as you see, this is all (mostly) documented and for sure doable in Java, but still not completely trivial.
I started a solution based on TinyTIFF, the answer from #haraldK on this SO question, the TIFF spec, and the Photoshop TIFF spec. It is about the simplest possible way to write a TIFF. I put in the code to write the Photoshop section, but it is not finished.
Note that Photoshop uses the TIFF image as the "preview" image, similar to the flattened composite image at the very end of a PSD file. The Photoshop TIFF section is what contains the pixel data for all the layers (again similar to a PSD). Adobe's use of TIFF in this way is pretty dirty. You might as well just use the (also terrible) PSD format, since smashing PSD data into the TIFF format just adds complexity for no benefit. This is why I did not finish the code below. If you do finish it, please post it here.
The Output class is from Kryo. pixmap.getPixels() is 4 bytes per pixel, RGBA.
/* Copyright (c) 2008-2015 Jan W. Krieger (<jan#jkrieger.de>, <j.krieger#dkfz.de>), German Cancer Research Center (DKFZ) & IWR, University of Heidelberg
* Copyright (c) 2018, Nathan Sweet, Esoteric Software LLC
* All rights reserved.
*
* This software is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public
* License (LGPL) as published by the Free Software Foundation, either version 2 of the License, or (at your option) any later
* version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You
* should have received a copy of the GNU General Public License along with this program. If not, see <http://www.gnu.org/licenses/>. */
public class TiffWriter {
private Output out;
private int width, height;
private int ifdCount, ifdLastOffset, ifdData, headerStart;
private Output header;
public void start (OutputStream output, int width, int height) throws IOException {
this.out = new Output(output);
this.width = width;
this.height = height;
out.writeByte('M'); // Big endian.
out.writeByte('M');
out.writeShort(42); // Magic number.
ifdLastOffset = out.total();
out.writeInt(8); // Offset of first IFD.
}
public void frame (Pixmap pixmap, String name, int frame, int endFrame) throws IOException {
ByteBuffer pixels = pixmap.getPixels();
headerStart = out.total();
ifdData = 2 + TIFF_HEADER_MAX_ENTRIES * 12;
ifdCount = 0;
header = new Output(TIFF_HEADER_SIZE + 2);
header.setPosition(2);
writeLongIFD(TIFF_FIELD_IMAGEWIDTH, width);
writeLongIFD(TIFF_FIELD_IMAGELENGTH, height);
writeShortIFD(TIFF_FIELD_BITSPERSAMPLE, 8, 8, 8);
writeShortIFD(TIFF_FIELD_COMPRESSION, COMPRESSION_NO);
writeShortIFD(TIFF_FIELD_PHOTOMETRICINTERPRETATION, PHOTOMETRIC_INTERPRETATION_RGB);
writeLongIFD(TIFF_FIELD_STRIPOFFSETS, headerStart + 2 + TIFF_HEADER_SIZE);
writeShortIFD(TIFF_FIELD_SAMPLESPERPIXEL, 4);
writeLongIFD(TIFF_FIELD_ROWSPERSTRIP, height);
writeLongIFD(TIFF_FIELD_STRIPBYTECOUNTS, width * height);
writeRationalIFD(TIFF_FIELD_XRESOLUTION, 720000, 10000);
writeRationalIFD(TIFF_FIELD_YRESOLUTION, 720000, 10000);
writeShortIFD(TIFF_FIELD_PLANARCONFIG, PLANAR_CONFIGURATION_CHUNKY);
writeShortIFD(TIFF_FIELD_RESOLUTIONUNIT, RESOLUTION_UNIT_INCH);
writeShortIFD(TIFF_FIELD_EXTRASAMPLES, 1); // Adds alpha to last samples per pixel.
// writeIFDEntrySHORT(TIFF_FIELD_SAMPLEFORMAT, SAMPLE_FORMAT_FLOAT);
// Photoshop layer entry.
ifdCount++;
header.writeShort(TIFF_FIELD_PHOTOSHOP_IMAGESOURCEDATA);
header.writeShort(TIFF_TYPE_UNDEFINED);
int sizePosition = header.position();
header.writeInt(0); // Size in bytes.
header.writeInt(ifdData + headerStart);
int pos = header.position();
header.setPosition(ifdData);
writeString(header, "Adobe Photoshop Document Data Block");
// Unfinished!
int size = header.position() - ifdData;
ifdData = header.position();
header.setPosition(sizePosition);
header.writeInt(size);
header.setPosition(pos);
if (ifdCount > TIFF_HEADER_MAX_ENTRIES) throw new RuntimeException();
header.setPosition(0);
header.writeShort(ifdCount);
header.setPosition(2 + ifdCount * 12); // header start + 12 bytes per IFD entry
header.writeInt(headerStart + 2 + TIFF_HEADER_SIZE + width * height);
out.writeBytes(header.getBuffer(), 0, TIFF_HEADER_SIZE + 2);
ifdLastOffset = headerStart + 2 + ifdCount * 12;
pixels.position(0);
for (int i = 0, n = width * height * 4; i < n; i += 4) {
byte a = pixels.get(i + 3);
float pma = (a & 0xff) / 255f;
out.writeByte((byte)((pixels.get(i) & 0xff) * pma));
out.writeByte((byte)((pixels.get(i + 1) & 0xff) * pma));
out.writeByte((byte)((pixels.get(i + 2) & 0xff) * pma));
out.writeByte(a);
}
pixels.position(0);
}
public void end () throws IOException {
out.close();
// Erase last IFD offset.
RandomAccessFile file = new RandomAccessFile("test.tif", "rw");
file.seek(ifdLastOffset);
file.write((byte)0);
file.write((byte)0);
file.write((byte)0);
file.write((byte)0);
file.close();
}
public void close () throws IOException {
end();
}
private void writeString (Output output, String value) {
for (int i = 0, n = value.length(); i < n; i++)
output.writeByte(value.charAt(i));
output.writeByte(0);
}
private void writeLongIFD (int tag, int data) {
ifdCount++;
header.writeShort(tag);
header.writeShort(TIFF_TYPE_LONG);
header.writeInt(1);
header.writeInt(data);
}
private void writeShortIFD (int tag, int data) {
ifdCount++;
header.writeShort(tag);
header.writeShort(TIFF_TYPE_SHORT);
header.writeInt(1);
header.writeShort(data);
header.writeShort(0); // Pad bytes.
}
private void writeShortIFD (int tag, int... data) {
ifdCount++;
header.writeShort(tag);
header.writeShort(TIFF_TYPE_SHORT);
header.writeInt(data.length);
if (data.length == 1)
header.writeInt(data[0]);
else {
header.writeInt(ifdData + headerStart);
int pos = header.position();
header.setPosition(ifdData);
for (int value : data)
header.writeShort(value);
ifdData = header.position();
header.setPosition(pos);
}
}
private void writeRationalIFD (int tag, int numerator, int denominator) {
ifdCount++;
header.writeShort(tag);
header.writeShort(TIFF_TYPE_RATIONAL);
header.writeInt(1);
header.writeInt(ifdData + headerStart);
int pos = header.position();
header.setPosition(ifdData);
header.writeInt(numerator);
header.writeInt(denominator);
ifdData = header.position();
header.setPosition(pos);
}
static private final int TIFF_HEADER_SIZE = 510;
static private final int TIFF_HEADER_MAX_ENTRIES = 16;
static private final int TIFF_FIELD_IMAGEWIDTH = 256;
static private final int TIFF_FIELD_IMAGELENGTH = 257;
static private final int TIFF_FIELD_BITSPERSAMPLE = 258;
static private final int TIFF_FIELD_COMPRESSION = 259;
static private final int TIFF_FIELD_PHOTOMETRICINTERPRETATION = 262;
static private final int TIFF_FIELD_IMAGEDESCRIPTION = 270;
static private final int TIFF_FIELD_STRIPOFFSETS = 273;
static private final int TIFF_FIELD_SAMPLESPERPIXEL = 277;
static private final int TIFF_FIELD_ROWSPERSTRIP = 278;
static private final int TIFF_FIELD_STRIPBYTECOUNTS = 279;
static private final int TIFF_FIELD_XRESOLUTION = 282;
static private final int TIFF_FIELD_YRESOLUTION = 283;
static private final int TIFF_FIELD_PLANARCONFIG = 284;
static private final int TIFF_FIELD_RESOLUTIONUNIT = 296;
static private final int TIFF_FIELD_EXTRASAMPLES = 338;
static private final int TIFF_FIELD_SAMPLEFORMAT = 339;
static private final int TIFF_FIELD_PHOTOSHOP_IMAGESOURCEDATA = 37724;
static private final int TIFF_TYPE_BYTE = 1;
static private final int TIFF_TYPE_ASCII = 2;
static private final int TIFF_TYPE_SHORT = 3;
static private final int TIFF_TYPE_LONG = 4;
static private final int TIFF_TYPE_RATIONAL = 5;
static private final int TIFF_TYPE_UNDEFINED = 7;
static private final int SAMPLE_FORMAT_UNSIGNED_INT = 1;
static private final int SAMPLE_FORMAT_SIGNED_INT = 2;
static private final int SAMPLE_FORMAT_FLOAT = 3;
static private final int SAMPLE_FORMAT_UNDEFINED = 4;
static private final int COMPRESSION_NO = 1;
static private final int COMPRESSION_CCITT_HUFFMAN = 2;
static private final int COMPRESSION_T4 = 3;
static private final int COMPRESSION_T6 = 4;
static private final int COMPRESSION_LZW = 5;
static private final int COMPRESSION_JPEG_OLD = 6;
static private final int COMPRESSION_JPEG_NEW = 7;
static private final int COMPRESSION_DEFLATE = 8;
static private final int PHOTOMETRIC_INTERPRETATION_WHITE_IS_ZERO = 0;
static private final int PHOTOMETRIC_INTERPRETATION_BLACK_IS_ZERO = 1;
static private final int PHOTOMETRIC_INTERPRETATION_RGB = 2;
static private final int PHOTOMETRIC_INTERPRETATION_PALETTE = 3;
static private final int PHOTOMETRIC_INTERPRETATION_TRANSPARENCY = 4;
static private final int PLANAR_CONFIGURATION_CHUNKY = 1;
static private final int PLANAR_CONFIGURATION_PLANAR = 2;
static private final int RESOLUTION_UNIT_NO = 1;
static private final int RESOLUTION_UNIT_INCH = 2;
static private final int RESOLUTION_UNIT_CENTIMETER = 3;
static public void main (String[] args) throws Exception {
FileOutputStream output = new FileOutputStream("test.tif");
TiffWriter writer = new TiffWriter();
writer.start(output, imageWidth, imageHeight);
for (int i = 0; i < 16; i++) {
Pixmap pixmap = new Pixmap(...);
writer.frame(pixmap, "run", i, 16);
}
writer.end();
writer.close();
}
}
I need to develop an app to record frequencies in real time using the phone's mic and then display them (in text). I am posting my code here. The FFT and complex classes have been used from http://introcs.cs.princeton.edu/java/97data/FFT.java.html and http://introcs.cs.princeton.edu/java/97data/Complex.java.html .The problem is when i run this on the emulator the frequency starts from some random value and keeps on increasing till 7996. It then repeats the whole process. Can someone plz help me out?
public class Main extends Activity {
TextView disp;
private static int[] sampleRate = new int[] { 44100, 22050, 11025, 8000 };
short audioData[];
double finalData[];
int bufferSize,srate;
String TAG;
public boolean recording;
AudioRecord recorder;
Complex[] fftArray;
float freq;
#Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
disp = (TextView) findViewById(R.id.display);
Thread t1 = new Thread(new Runnable(){
public void run() {
Log.i(TAG,"Setting up recording");
for (int rate : sampleRate) {
try{
Log.d(TAG, "Attempting rate " + rate);
bufferSize=AudioRecord.getMinBufferSize(rate,AudioFormat.CHANNEL_CONFIGURATION_MONO,
AudioFormat.ENCODING_PCM_16BIT)*3; //get the buffer size to use with this audio record
if (bufferSize != AudioRecord.ERROR_BAD_VALUE) {
recorder = new AudioRecord (MediaRecorder.AudioSource.MIC,rate,AudioFormat.CHANNEL_CONFIGURATION_MONO,
AudioFormat.ENCODING_PCM_16BIT,2048); //instantiate the AudioRecorder
Log.d(TAG, "BufferSize " +bufferSize);
srate = rate;
}
} catch (Exception e) {
Log.e(TAG, rate + "Exception, keep trying.",e);
}
}
bufferSize=2048;
recording=true; //variable to use start or stop recording
audioData = new short [bufferSize]; //short array that pcm data is put into.
Log.i(TAG,"Got buffer size =" + bufferSize);
while (recording) { //loop while recording is needed
Log.i(TAG,"in while 1");
if (recorder.getState()==android.media.AudioRecord.STATE_INITIALIZED) // check to see if the recorder has initialized yet.
if (recorder.getRecordingState()==android.media.AudioRecord.RECORDSTATE_STOPPED)
recorder.startRecording(); //check to see if the Recorder has stopped or is not recording, and make it record.
else {
Log.i(TAG,"in else");
// audiorecord();
finalData=convert_to_double(audioData);
Findfft();
for(int k=0;k<fftArray.length;k++)
{
freq = ((float)srate/(float) fftArray.length) *(float)k;
runOnUiThread(new Runnable(){
public void run()
{
disp.setText("The frequency is " + freq);
if(freq>=15000)
recording = false;
}
});
}
}//else recorder started
} //while recording
if (recorder.getState()==android.media.AudioRecord.RECORDSTATE_RECORDING)
recorder.stop(); //stop the recorder before ending the thread
recorder.release(); //release the recorders resources
recorder=null; //set the recorder to be garbage collected.
}//run
});
t1.start();
}
private void Findfft() {
// TODO Auto-generated method stub
Complex[] fftTempArray = new Complex[bufferSize];
for (int i=0; i<bufferSize; i++)
{
fftTempArray[i] = new Complex(finalData[i], 0);
}
fftArray = FFT.fft(fftTempArray);
}
private double[] convert_to_double(short data[]) {
// TODO Auto-generated method stub
double[] transformed = new double[data.length];
for (int j=0;j<data.length;j++) {
transformed[j] = (double)data[j];
}
return transformed;
}
#Override
public boolean onCreateOptionsMenu(Menu menu) {
// Inflate the menu; this adds items to the action bar if it is present.
getMenuInflater().inflate(R.menu.main, menu);
return true;
}
}
Your question has been succinctly answered, however, to further your objectives and complete the loop...
Yes, FFT is not optimal on limited CPUs for pitch / frequency identification. A more optimal approach is YIN described here. You may find an implementation at Tarsos.
Issues you will face are the lack of javax.sound.sampled in the ADK and therefore converting the shorts/bytes from AudioRecord to the floats required for the referenced implementations.
Your problem is right here:
Findfft();
for(int k=0;k<fftArray.length;k++) {
freq = ((float)srate/(float) fftArray.length) *(float)k;
runOnUiThread(new Runnable() {
public void run() {
disp.setText("The frequency is " + freq);
if(freq>=15000) recording = false;
}
});
}
All this for loop does is go through your array of FFT values, convert the array index to a frequency in Hz, and print it.
If you want to output what frequency you're recording, you should at least look at the data in your array - the crudest method would be to calculate the square real magnitude and find the frequency bin with the biggest.
In addition to that, I don't think the FFT algorithm you're using does any precalculations - there are others that do, and seeing as you're developing for a mobile device, you might want to take CPU usage and power use into account.
JTransforms is one library that does use precalculation to lower CPU load, and its documentation is very complete.
You may also find useful information on how to interpret the data returned from the FFT at Wikipedia - no offense, but it looks like you're not quite sure what you're doing, so I'm giving pointers.
Lastly, if you're looking to use this app for musical notes, I seem to remember lots of people saying that an FFT isn't the best way to do that, but I can't remember what is. Maybe someone else can add that bit?
i find this solution after few days - the Best for getting frequency in Hrz:
download Jtransforms and this Jar also - Jtransforms need it.
then i use this task:
public class MyRecorder extends AsyncTask<Void, short[], Void> {
int blockSize = 2048;// = 256;
private static final int RECORDER_SAMPLERATE = 8000;
private static final int RECORDER_CHANNELS = AudioFormat.CHANNEL_IN_MONO;
private static final int RECORDER_AUDIO_ENCODING = AudioFormat.ENCODING_PCM_16BIT;
int BufferElements2Rec = 1024; // want to play 2048 (2K) since 2 bytes we use only 1024
int BytesPerElement = 2;
#Override
protected Void doInBackground(Void... params) {
try {
final AudioRecord audioRecord = new AudioRecord(MediaRecorder.AudioSource.MIC,
RECORDER_SAMPLERATE, RECORDER_CHANNELS,
RECORDER_AUDIO_ENCODING, BufferElements2Rec * BytesPerElement);
if (audioRecord == null) {
return null;
}
final short[] buffer = new short[blockSize];
final double[] toTransform = new double[blockSize];
audioRecord.startRecording();
while (started) {
Thread.sleep(100);
final int bufferReadResult = audioRecord.read(buffer, 0, blockSize);
publishProgress(buffer);
}
audioRecord.stop();
audioRecord.release();
} catch (Throwable t) {
Log.e("AudioRecord", "Recording Failed");
}
return null;
}
#Override
protected void onProgressUpdate(short[]... buffer) {
super.onProgressUpdate(buffer);
float freq = calculate(RECORDER_SAMPLERATE, buffer[0]);
}
public static float calculate(int sampleRate, short [] audioData)
{
int numSamples = audioData.length;
int numCrossing = 0;
for (int p = 0; p < numSamples-1; p++)
{
if ((audioData[p] > 0 && audioData[p + 1] <= 0) ||
(audioData[p] < 0 && audioData[p + 1] >= 0))
{
numCrossing++;
}
}
float numSecondsRecorded = (float)numSamples/(float)sampleRate;
float numCycles = numCrossing/2;
float frequency = numCycles/numSecondsRecorded;
return frequency;
}
i am writing a program on smartphone (on Android)
It is about :
Analyzing spectrum of sound by fft algorithms
measuring the intensity of a sound have f = fo (ex. fo = 18khz) from the spectrum which I have got results from the analysis above.
Calculating the distance from smartphone to source of sound with this intensity
After fft, I got two arrays (real and image). I calculate the sound intensity at f=18000hz( suppose that source of sound at 18000 hz is unchanged so that it makes it easier to measure sound intensity). As follow:
frequency at bin FFT[i] is :
if i <= [N/2] then i * SamplingFrequency / N
if i >= [N/2] then (N-i) * SamplingFrequency / N
therefore at frequency = 18000hz then I choose i = 304
sound intensity = real_array[304] * real_array[304] + image_array[304] * image_array[304]
However, the intensity, in fact, varies a lot making it difficult to measure the distance. And, I have no idea how to explain this.
Besides, I would like to ask you a question that the intensity I have measured above uses what unit to calculate.
Here is my code:
a. fft algorithms( I use fft 512 point)
import define.define512;
public class fft {
private static float[] W_real;
private static float[] W_img;
private static float[] input_real= new float[512];
private static float[] input_img;
//input_real1 is values from mic(smartphone)
//output is values of sound intensity
public static void FFT(float[] input_real1, float[] output)
{
for(int i =0;i<512;i++) input_real[i] = input_real1[i];
input_img = new float[512];
W_real = define512.W_IMAG;
W_img = define512.W_IMAG;
int[] W_order = define512.ORDER;
float[] output_real = new float[512], output_img = new float[512];
fftradix2(0,511);
//reorder deals with inverse bit
reorder(input_real, input_img, output_real, output_img, W_order, 512);
for(int i =0;i<512;i++)
{
output[i] = sqrt((output_real[i]*output_real[i] + output_img[i]*output_img[i]));
}
}
private static void reorder(float[] in_real,float[] in_imag, float[] out_real,float[] out_imag,int[] order,int N){
for(int i=0;i<N;i++){
out_real[i]=in_real[order[i]];
out_imag[i]=in_imag[order[i]];
}
}
//fft algorithms
private static void fftradix2(int dau,int cuoi)
{
int check = cuoi - dau;
if (check == 1)
{
input_real[dau] = input_real[dau] + input_real[cuoi];
input_img[dau] = input_img[dau] + input_img[cuoi];
input_real[cuoi] = input_real[dau] -2* input_real[cuoi];
input_img[cuoi] = input_img[dau] -2* input_img[cuoi];
}
else
{
int index = 512/(cuoi - dau + 1);
int tg = (cuoi - dau)/2;
fftradix2(dau,(dau+tg));
fftradix2((cuoi-tg),cuoi);
for(int i = dau;i<=(dau+tg);i++)
{
input_real[i] = input_real[i] + input_real[i+tg+1]*W_real[(i-dau)*index] - input_img[i+tg+1]*W_img[(i-dau)*index];
input_img[i] = input_img[i] + input_real[i+tg+1]*W_img[(i-dau)*index] + input_img[i+tg+1]*W_real[(i%(tg+1))*index];
input_real[i+tg+1] = input_real[i] -2* input_real[i+tg+1]*W_real[(i-dau)*index] +2* input_img[i+tg+1]*W_img[(i-dau)*index];
input_img[i+tg+1] = input_img[i] -2* input_real[i+tg+1]*W_img[(i-dau)*index] -2* input_img[i+tg+1]*W_real[(i-dau)*index];
}
}
}
}
b. code use mic in smartphone
NumOverlapSample = 800;
NumNewSample = 224;
private static int Fs = 44100;
private byte recorderAudiobuffer[] = new byte [1024];
AudioRecord recorder = new AudioRecord(AudioSource.MIC, Fs, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, 4096);
//start recorder
recorder.startRecording();
timer.schedule(new task_update(), 1000, 10);
class task_update extends TimerTask
{
#Override
public void run() {
// TODO Auto-generated method stub
for(int i=0;i<NumOverlapSample;i++)
recorderAudiobuffer[i] = recorderAudiobuffer[i+NumNewSample];
int bufferRead = recorder.read(recorderAudiobuffer,NumOverlapSample,NumNewSample);
convert.decode(recorderAudiobuffer, N, input);
fft.FFT(input, output);
}
and my soucre https://www.box.com/s/zuppzkicymfsuv4kb65p
thanks for all
At 18 kHz, microphone type, position and direction, as well as sound reflections from the nearby acoustic environment will strongly influence the sound level.