Develop Reference

Implementing custom prefix remover token filter in lucene producing dirty tokens

i'm trying to implement a lucene filter to remove a prefix from a term in a query.
It seems that sometime after multiple queries, the filter has been reused so the char buffer is dirty.
Code below is simplified, prefix is an external parameter.
public static class PrefixFilter extends TokenFilter {
private final PackedTokenAttributeImpl termAtt = (PackedTokenAttributeImpl) addAttribute(CharTermAttribute.class);
public PrefixFilter(TokenStream in) {
super(in);
}
#Override
public final boolean incrementToken() throws IOException {
if (!input.incrementToken()) {
return false;
}
String value = new String(termAtt.buffer());
value = value.trim();
value = value.toLowerCase();
value = StringUtils.removeStart(value, "prefix_");
if (value.isBlank()) {
termAtt.setEmpty();
} else {
termAtt.copyBuffer(value.toCharArray(), 0, value.length());
termAtt.setLength(value.length());
}
return true;
}
}
So after 10 or twelve queries, the value "prefix_a" became "abcde".
So i'm trying to add termBuffer offset end value in this way:
termAtt.setEmpty();
termAtt.resizeBuffer(value.length());
termAtt.copyBuffer(value.toCharArray(), 0, value.length());
termAtt.setLength(value.length());
termAtt.setOffset(0, value.length());
But i don't know if it's correct. Can anyone help me?
Thanks.

See if this helps you,
/**
* Standard number token filter.
*/
public class StandardnumberTokenFilter extends TokenFilter {
private final LinkedList<PackedTokenAttributeImpl> tokens;
private final StandardnumberService service;
private final Settings settings;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
private State current;
protected StandardnumberTokenFilter(TokenStream input, StandardnumberService service, Settings settings) {
super(input);
this.tokens = new LinkedList<>();
this.service = service;
this.settings = settings;
}
#Override
public final boolean incrementToken() throws IOException {
if (!tokens.isEmpty()) {
if (current == null) {
throw new IllegalArgumentException("current is null");
}
PackedTokenAttributeImpl token = tokens.removeFirst();
restoreState(current);
termAtt.setEmpty().append(token);
posIncAtt.setPositionIncrement(0);
return true;
}
if (input.incrementToken()) {
detect();
if (!tokens.isEmpty()) {
current = captureState();
}
return true;
} else {
return false;
}
}
private void detect() throws CharacterCodingException {
CharSequence term = new String(termAtt.buffer(), 0, termAtt.length());
Collection<CharSequence> variants = service.lookup(settings, term);
for (CharSequence ch : variants) {
if (ch != null) {
PackedTokenAttributeImpl token = new PackedTokenAttributeImpl();
token.append(ch);
tokens.add(token);
}
}
}
#Override
public void reset() throws IOException {
super.reset();
tokens.clear();
current = null;
}
#Override
public boolean equals(Object object) {
return object instanceof StandardnumberTokenFilter &&
service.equals(((StandardnumberTokenFilter)object).service) &&
settings.equals(((StandardnumberTokenFilter)object).settings);
}
#Override
public int hashCode() {
return service.hashCode() ^ settings.hashCode();
}
}
https://github.com/jprante/elasticsearch-plugin-bundle/blob/f63690f877cc7f50360faffbac827622c9d404ef/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/analysis/standardnumber/StandardnumberTokenFilter.java

Global combine not producing output Apache Beam

I am trying to write an unbounded ping pipeline that takes output from a ping command and parses it to determine some statistics about the RTT (avg/min/max) and for now, just print the results.
I have already written an unbounded ping source that outputs each line as it comes in. The results are windowed every second for every 5 seconds of pings. The windowed data is fed to a Combine.globally call to statefully process the string outputs. The problem is that the accumulators are never merged and the output is never extracted. This means that the pipeline never continues past this point. What am I doing wrong here?
public class TestPingIPs {
public static void main(String[] args)
{
PipelineOptions options = PipelineOptionsFactory.create();
Pipeline pipeline = Pipeline.create(options);
String destination = "8.8.8.8";
PCollection<PingResult> res =
/*
Run the unbounded ping command. Only the lines where the result of the ping command are returned.
No statistics or first startup lines are returned here.
*/
pipeline.apply("Ping command",
PingCmd.read()
.withPingArguments(PingCmd.PingArguments.create(destination, -1)))
/*
Window the ping command strings into 5 second sliding windows produced every 1 second
*/
.apply("Window strings",
Window.into(SlidingWindows.of(Duration.standardSeconds(5))
.every(Duration.standardSeconds(1))))
/*
Parse and aggregate the strings into a PingResult object using stateful processing.
*/
.apply("Combine the pings",
Combine.globally(new ProcessPings()).withoutDefaults())
/*
Test our output to see what we get here
*/
.apply("Test output",
ParDo.of(new DoFn<PingResult, PingResult>() {
#ProcessElement
public void processElement(ProcessContext c)
{
System.out.println(c.element().getAvgRTT());
System.out.println(c.element().getPacketLoss());
c.output(c.element());
}
}));
pipeline.run().waitUntilFinish();
}
static class ProcessPings extends Combine.CombineFn<String, RttStats, PingResult> {
private long getRTTFromLine(String line){
long rtt = Long.parseLong(line.split("time=")[1].split("ms")[0]);
return rtt;
}
#Override
public RttStats createAccumulator()
{
return new RttStats();
}
#Override
public RttStats addInput(RttStats mutableAccumulator, String input)
{
mutableAccumulator.incTotal();
if (input.contains("unreachable")) {
_unreachableCount.inc();
mutableAccumulator.incPacketLoss();
}
else if (input.contains("General failure")) {
_transmitFailureCount.inc();
mutableAccumulator.incPacketLoss();
}
else if (input.contains("timed out")) {
_timeoutCount.inc();
mutableAccumulator.incPacketLoss();
}
else if (input.contains("could not find")) {
_unknownHostCount.inc();
mutableAccumulator.incPacketLoss();
}
else {
_successfulCount.inc();
mutableAccumulator.add(getRTTFromLine(input));
}
return mutableAccumulator;
}
#Override
public RttStats mergeAccumulators(Iterable<RttStats> accumulators)
{
Iterator<RttStats> iter = accumulators.iterator();
if (!iter.hasNext()){
return createAccumulator();
}
RttStats running = iter.next();
while (iter.hasNext()){
RttStats next = iter.next();
running.addAll(next.getVals());
running.addLostPackets(next.getLostPackets());
}
return running;
}
#Override
public PingResult extractOutput(RttStats stats)
{
stats.calculate();
boolean connected = stats.getPacketLoss() != 1;
return new PingResult(connected, stats.getAvg(), stats.getMin(), stats.getMax(), stats.getPacketLoss());
}
private final Counter _successfulCount = Metrics.counter(ProcessPings.class, "Successful pings");
private final Counter _unknownHostCount = Metrics.counter(ProcessPings.class, "Unknown hosts");
private final Counter _transmitFailureCount = Metrics.counter(ProcessPings.class, "Transmit failures");
private final Counter _timeoutCount = Metrics.counter(ProcessPings.class, "Timeouts");
private final Counter _unreachableCount = Metrics.counter(ProcessPings.class, "Unreachable host");
}
I would guess that there are some issues with the CombineFn that I wrote, but I can't seem to figure out what's going wrong here! I tried following the example here, but there's still something I must be missing.
EDIT: I added the ping command implementation below. This is running on a Direct Runner while I test.
PingCmd.java:
public class PingCmd {
public static Read read(){
if (System.getProperty("os.name").startsWith("Windows")) {
return WindowsPingCmd.read();
}
else{
return null;
}
}
WindowsPingCmd.java:
public class WindowsPingCmd extends PingCmd {
private WindowsPingCmd()
{
}
public static PingCmd.Read read()
{
return new WindowsRead.Builder().build();
}
static class PingCheckpointMark implements UnboundedSource.CheckpointMark, Serializable {
#VisibleForTesting
Instant oldestMessageTimestamp = Instant.now();
#VisibleForTesting
transient List<String> outputs = new ArrayList<>();
public PingCheckpointMark()
{
}
public void add(String message, Instant timestamp)
{
if (timestamp.isBefore(oldestMessageTimestamp)) {
oldestMessageTimestamp = timestamp;
}
outputs.add(message);
}
#Override
public void finalizeCheckpoint()
{
oldestMessageTimestamp = Instant.now();
outputs.clear();
}
// set an empty list to messages when deserialize
private void readObject(java.io.ObjectInputStream stream)
throws IOException, ClassNotFoundException
{
stream.defaultReadObject();
outputs = new ArrayList<>();
}
#Override
public boolean equals(#Nullable Object other)
{
if (other instanceof PingCheckpointMark) {
PingCheckpointMark that = (PingCheckpointMark) other;
return Objects.equals(this.oldestMessageTimestamp, that.oldestMessageTimestamp)
&& Objects.deepEquals(this.outputs, that.outputs);
}
else {
return false;
}
}
}
#VisibleForTesting
static class UnboundedPingSource extends UnboundedSource<String, PingCheckpointMark> {
private final WindowsRead spec;
public UnboundedPingSource(WindowsRead spec)
{
this.spec = spec;
}
#Override
public UnboundedReader<String> createReader(
PipelineOptions options, PingCheckpointMark checkpointMark)
{
return new UnboundedPingReader(this, checkpointMark);
}
#Override
public List<UnboundedPingSource> split(int desiredNumSplits, PipelineOptions options)
{
// Don't really need to ever split the ping source, so we should just have one per destination
return Collections.singletonList(new UnboundedPingSource(spec));
}
#Override
public void populateDisplayData(DisplayData.Builder builder)
{
spec.populateDisplayData(builder);
}
#Override
public Coder<PingCheckpointMark> getCheckpointMarkCoder()
{
return SerializableCoder.of(PingCheckpointMark.class);
}
#Override
public Coder<String> getOutputCoder()
{
return StringUtf8Coder.of();
}
}
#VisibleForTesting
static class UnboundedPingReader extends UnboundedSource.UnboundedReader<String> {
private final UnboundedPingSource source;
private String current;
private Instant currentTimestamp;
private final PingCheckpointMark checkpointMark;
private BufferedReader processOutput;
private Process process;
private boolean finishedPings;
private int maxCount = 5;
private static AtomicInteger currCount = new AtomicInteger(0);
public UnboundedPingReader(UnboundedPingSource source, PingCheckpointMark checkpointMark)
{
this.finishedPings = false;
this.source = source;
this.current = null;
if (checkpointMark != null) {
this.checkpointMark = checkpointMark;
}
else {
this.checkpointMark = new PingCheckpointMark();
}
}
#Override
public boolean start() throws IOException
{
WindowsRead spec = source.spec;
String cmd = createCommand(spec.pingConfiguration().getPingCount(), spec.pingConfiguration().getDestination());
try {
ProcessBuilder builder = new ProcessBuilder(cmd.split(" "));
builder.redirectErrorStream(true);
process = builder.start();
processOutput = new BufferedReader(new InputStreamReader(process.getInputStream()));
return advance();
} catch (Exception e) {
throw new IOException(e);
}
}
private String createCommand(int count, String dest){
StringBuilder builder = new StringBuilder("ping");
String countParam = "";
if (count <= 0){
countParam = "-t";
}
else{
countParam += "-n " + count;
}
return builder.append(" ").append(countParam).append(" ").append(dest).toString();
}
#Override
public boolean advance() throws IOException
{
String line = processOutput.readLine();
// Ignore empty/null lines
if (line == null || line.isEmpty()) {
line = processOutput.readLine();
}
// Ignore the 'Pinging <dest> with 32 bytes of data' line
if (line.contains("Pinging " + source.spec.pingConfiguration().getDestination())) {
line = processOutput.readLine();
}
// If the pings have finished, ignore
if (finishedPings) {
return false;
}
// If this is the start of the statistics, the pings are done and we can just exit
if (line.contains("statistics")) {
finishedPings = true;
}
current = line;
currentTimestamp = Instant.now();
checkpointMark.add(current, currentTimestamp);
if (currCount.incrementAndGet() == maxCount){
currCount.set(0);
return false;
}
return true;
}
#Override
public void close() throws IOException
{
if (process != null) {
process.destroy();
if (process.isAlive()) {
process.destroyForcibly();
}
}
}
#Override
public Instant getWatermark()
{
return checkpointMark.oldestMessageTimestamp;
}
#Override
public UnboundedSource.CheckpointMark getCheckpointMark()
{
return checkpointMark;
}
#Override
public String getCurrent()
{
if (current == null) {
throw new NoSuchElementException();
}
return current;
}
#Override
public Instant getCurrentTimestamp()
{
if (current == null) {
throw new NoSuchElementException();
}
return currentTimestamp;
}
#Override
public UnboundedPingSource getCurrentSource()
{
return source;
}
}
public static class WindowsRead extends PingCmd.Read {
private final PingArguments pingConfig;
private WindowsRead(PingArguments pingConfig)
{
this.pingConfig = pingConfig;
}
public Builder builder()
{
return new WindowsRead.Builder(this);
}
PingArguments pingConfiguration()
{
return pingConfig;
}
public WindowsRead withPingArguments(PingArguments configuration)
{
checkArgument(configuration != null, "configuration can not be null");
return builder().setPingArguments(configuration).build();
}
#Override
public PCollection<String> expand(PBegin input)
{
org.apache.beam.sdk.io.Read.Unbounded<String> unbounded =
org.apache.beam.sdk.io.Read.from(new UnboundedPingSource(this));
return input.getPipeline().apply(unbounded);
}
#Override
public void populateDisplayData(DisplayData.Builder builder)
{
super.populateDisplayData(builder);
pingConfiguration().populateDisplayData(builder);
}
static class Builder {
private PingArguments config;
Builder()
{
}
private Builder(WindowsRead source)
{
this.config = source.pingConfiguration();
}
WindowsRead.Builder setPingArguments(PingArguments config)
{
this.config = config;
return this;
}
WindowsRead build()
{
return new WindowsRead(this.config);
}
}
#Override
public int hashCode()
{
return Objects.hash(pingConfig);
}
}

One thing I notice in your code is that advance() always returns True. The watermark only advances on bundle completion, and I think it's runner-dependent whether a runner will ever complete a bundle if advance ever never returns False. You could try returning False after a bounded amount of time/number of pings.
You could also consider re-writing this as an SDF.

Copy the list of pojo to clipboard

Lang: Java
We are trying to copy the list of pojo to the clipboard in the table format.
Bullet point: what we are trying to achieve here :
1. convert list of pojo into table format
2. If the user copy it in some excel sheet then the it should be copied easily or even if the user try to copy in notepad it should print in the table format.
3. Add some meta data to clipboard to determine the pojo when we will import the table again.
For converting the list of pojo to table format i have used the jtable but i am not able to export all the jtable content to clipboard.
can anyone suggest if i should follow the jtable approach and copy the table to clipboard or any other solution is also available.
Update: as suggested in the comment I tried using the flavours
public class ClipboardTest implements ClipboardOwner {
public static void main(String[] args) {
ClipboardTest clipboardTest = new ClipboardTest();
clipboardTest.copyToClipboard();
//clipboardTest.getFromClipboard();
}
public void copyToClipboard() {
Clipboard clipboard = Toolkit.getDefaultToolkit().getSystemClipboard();
Pojo data = new Pojo("1", "2", "ame2", "2", "2");
MyObjectSelection dataSelection = new MyObjectSelection(data);
StringSelection selection = new StringSelection("testing string");
clipboard.setContents(dataSelection, ClipboardTest.this);
System.out.println("copied to clipboard");
}
public void getFromClipboard() {
Clipboard clipboard = Toolkit.getDefaultToolkit().getSystemClipboard();
Transferable clipboardContent = clipboard.getContents(this);
DataFlavor[] flavors = clipboardContent.getTransferDataFlavors();
System.out.println("flavors.length = " + flavors.length);
for (int i = 0; i < flavors.length; i++) {
System.out.println("flavor[" + i + "] = " + flavors[i]);
}
}
// ClipboardOwner implementation
#Override
public void lostOwnership(Clipboard clipboard, Transferable transferable) {
System.out.println("ClipboardTest: Lost ownership");
}
}
---
myobjectselection.java
public class MyObjectSelection implements Transferable, ClipboardOwner {
private static DataFlavor dmselFlavor = new DataFlavor(Pojo.class,
"Test data flavor");
private Pojo selection;
public MyObjectSelection(Pojo selection) {
this.selection = selection;
}
// Transferable implementation
#Override
public DataFlavor[] getTransferDataFlavors() {
System.out.println("getTransferDataFlavors");
DataFlavor[] ret = { dmselFlavor };
return ret;
}
#Override
public boolean isDataFlavorSupported(DataFlavor flavor) {
return dmselFlavor.equals(flavor);
}
#Override
public synchronized Object getTransferData(DataFlavor flavor)
throws UnsupportedFlavorException {
if (isDataFlavorSupported(flavor)) {
return this.selection;
} else {
throw new UnsupportedFlavorException(dmselFlavor);
}
}
// ClipboardOwner implementation
#Override
public void lostOwnership(Clipboard clipboard, Transferable transferable) {
System.out.println("MyObjectSelection: Lost ownership");
}
}
--
pojo.java
public class Pojo implements Serializable{
/**
*
*/
private static final long serialVersionUID = 1L;
private String name;
private String name1;
private String name2;
private String name3;
private String name4;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getName1() {
return name1;
}
public void setName1(String name1) {
this.name1 = name1;
}
public String getName2() {
return name2;
}
public void setName2(String name2) {
this.name2 = name2;
}
public String getName3() {
return name3;
}
public void setName3(String name3) {
this.name3 = name3;
}
public String getName4() {
return name4;
}
public void setName4(String name4) {
this.name4 = name4;
}
public Pojo(String name, String name1, String name2, String name3,
String name4) {
super();
this.name = name;
this.name1 = name1;
this.name2 = name2;
this.name3 = name3;
this.name4 = name4;
}
}
When i am trying to copy the string value to clipboard it is working but when i am trying to copy the pojo then is it not working.

For every flavor you want to support, you must provide methods to encode the object in the specified format.
This means you'll likely need to provide an encoder for plain text, html and CVS to cover the basics.
So, based on your Pojo, I wrote a Transferable that supports:
List (of Pojos)
HTML
CVS
Plain text
And serialised (as an additional to List, but it's essentially the same)
PojoTransferable
public class PojoTransferable implements Transferable {
public static final DataFlavor POJO_LIST_DATA_FLAVOR = new DataFlavor(List.class, "application/x-java-pojo-list;class=java.util.List");
public static final DataFlavor HTML_DATA_FLAVOR = new DataFlavor("text/html", "HTML");
public static final DataFlavor CSV_DATA_FLAVOR = new DataFlavor("text/csv", "CVS");
public static final DataFlavor PLAIN_DATA_FLAVOR = new DataFlavor("text/plain", "Plain text");
public static final DataFlavor SERIALIZED_DATA_FLAVOR = new DataFlavor(Pojo.class, "application/x-java-serialized-object; Pojo");
private static String[] HEADERS = new String[]{"name", "name1", "name2", "name3", "name4"};
private static Pojo POJO_HEADER = new Pojo("name", "name1", "name2", "name3", "name4");
private List<Pojo> pojos;
public PojoTransferable(List<Pojo> pojos) {
this.pojos = pojos;
}
#Override
public DataFlavor[] getTransferDataFlavors() {
return new DataFlavor[]{POJO_LIST_DATA_FLAVOR, HTML_DATA_FLAVOR, CSV_DATA_FLAVOR, SERIALIZED_DATA_FLAVOR, PLAIN_DATA_FLAVOR};
}
#Override
public boolean isDataFlavorSupported(DataFlavor flavor) {
boolean supported = false;
for (DataFlavor mine : getTransferDataFlavors()) {
if (mine.equals(flavor)) {
supported = true;
break;
}
}
return supported;
}
#Override
public Object getTransferData(DataFlavor flavor) throws UnsupportedFlavorException, IOException {
Object data = null;
if (POJO_LIST_DATA_FLAVOR.equals(flavor)) {
data = pojos;
} else if (HTML_DATA_FLAVOR.equals(flavor)) {
data = new ByteArrayInputStream(formatAsHTML().getBytes());
} else if (SERIALIZED_DATA_FLAVOR.equals(flavor)) {
data = pojos;
} else if (CSV_DATA_FLAVOR.equals(flavor)) {
data = new ByteArrayInputStream(formatAsCVS().getBytes());
} else if (PLAIN_DATA_FLAVOR.equals(flavor)) {
data = new ByteArrayInputStream(formatAsPlainText().getBytes());
} else {
throw new UnsupportedFlavorException(flavor);
}
return data;
}
protected String formatAsCVS(Pojo pojo) {
StringJoiner sj = new StringJoiner(",");
sj.add(pojo.getName());
sj.add(pojo.getName2());
sj.add(pojo.getName3());
sj.add(pojo.getName4());
return sj.toString();
}
public String formatAsCVS() {
StringBuilder sb = new StringBuilder(128);
sb.append(formatAsCVS(POJO_HEADER));
for (Pojo pojo : pojos) {
sb.append(formatAsCVS(pojo));
}
return "";
}
protected Map<Integer, Integer> columnWidthsFor(Pojo pojo) {
Map<Integer, Integer> columnWidths = new HashMap<>();
columnWidths.put(0, pojo.getName().length());
columnWidths.put(1, pojo.getName1().length());
columnWidths.put(2, pojo.getName2().length());
columnWidths.put(3, pojo.getName3().length());
columnWidths.put(4, pojo.getName4().length());
return columnWidths;
}
protected void apply(Map<Integer, Integer> pojoWidths, Map<Integer, Integer> columnWidths) {
for (int index = 0; index < 5; index++) {
int currentWidth = 2;
if (columnWidths.containsKey(index)) {
currentWidth = columnWidths.get(index);
}
int columnWidth = 2;
if (pojoWidths.containsKey(index)) {
columnWidth = pojoWidths.get(index);
}
columnWidths.put(index, Math.max(currentWidth, columnWidth));
}
}
protected String formatAsPlainText(Pojo pojo, String format) {
return String.format(format, pojo.getName(), pojo.getName1(), pojo.getName2(), pojo.getName3(), pojo.getName4());
}
public static String fill(int padding) {
return String.format("%" + padding + "s", "").replace(" ", "-");
}
public String formatAsPlainText() {
Map<Integer, Integer> columnWidths = new HashMap<>();
apply(columnWidthsFor(POJO_HEADER), columnWidths);
for (Pojo pojo : pojos) {
apply(columnWidthsFor(pojo), columnWidths);
}
StringJoiner sjFormat = new StringJoiner("|");
StringJoiner sjSep = new StringJoiner("+");
for (int index = 0; index < 5; index++) {
int currentWidth = 0;
if (columnWidths.containsKey(index)) {
currentWidth = columnWidths.get(index);
}
sjFormat.add(" %-" + currentWidth + "s ");
sjSep.add(fill(currentWidth + 2));
}
sjFormat.add("%n");
sjSep.add("\n");
String seperator = sjSep.toString();
String format = sjFormat.toString();
StringBuilder sb = new StringBuilder(128);
sb.append(formatAsPlainText(POJO_HEADER, format));
for (Pojo pojo : pojos) {
sb.append(seperator);
sb.append(formatAsPlainText(pojo, format));
}
return sb.toString();
}
public String formatAsHTML() {
StringBuilder sb = new StringBuilder(128);
sb.append("<html><body>");
sb.append("<table border='1'>");
sb.append("<tr>");
for (String header : HEADERS) {
sb.append("<th>").append(header).append("</th>");
}
sb.append("</tr>");
for (Pojo pojo : pojos) {
sb.append("<tr>");
sb.append("<td>").append(pojo.getName()).append("</td>");
sb.append("<td>").append(pojo.getName1()).append("</td>");
sb.append("<td>").append(pojo.getName2()).append("</td>");
sb.append("<td>").append(pojo.getName3()).append("</td>");
sb.append("<td>").append(pojo.getName4()).append("</td>");
sb.append("</tr>");
}
sb.append("</table>");
return sb.toString();
}
}
Test...
Then I wrote a really simple test...
List<Pojo> pojos = new ArrayList<>(25);
pojos.add(new Pojo("one", "two", "three", "four", "five"));
PojoTransferable pt = new PojoTransferable(pojos);
Clipboard cb = Toolkit.getDefaultToolkit().getSystemClipboard();
cb.setContents(pt, new ClipboardOwner() {
#Override
public void lostOwnership(Clipboard clipboard, Transferable contents) {
System.out.println("Lost");
}
});
try {
Object data = cb.getData(PojoTransferable.POJO_LIST_DATA_FLAVOR);
if (data instanceof List) {
List listOfPojos = (List)data;
System.out.println("listOfPojos contains " + listOfPojos.size());
for (Object o : listOfPojos) {
System.out.println(o);
}
}
} catch (UnsupportedFlavorException ex) {
Logger.getLogger(Test.class.getName()).log(Level.SEVERE, null, ex);
} catch (IOException ex) {
Logger.getLogger(Test.class.getName()).log(Level.SEVERE, null, ex);
}
When I run it, it prints out...
listOfPojos contains 1
test.Pojo#5b480cf9
which tells use that the List flavor worked (for our needs), but then I opened up a text edit, Word and Excel and pasted the contents into them...
nb: Excel used the HTML formatting, go figure

ANTLR4 parse tree to DOT using DOTGenerator

How do I use DOTGenerator to convert a parse tree to DOT/graphviz format in ANTLR4?
I found this related question but the only answer uses TreeViewer to display the tree in a JPanel and that's not what I'm after. This other question is exacly what I need but it didn't get answered. Everything else I stumbled upon relates to DOTTreeGenerator from ANTLR3 and it's not helpful.
I'm using Java with the ANTLR4 plugin for IntelliJ.

I have a small project that has all kind of utility methods w.r.t. ANTLR4 grammar debugging/testing. I haven't found the time to provide it of some proper documentation so that I can put it on Github. But here's a part of it responsible for creating a DOT file from a grammar.
Stick it all in a single file called Main.java (and of course generate the lexer and parser for Expression.g4), and you will see a DOT string being printed to your console:
import org.antlr.v4.runtime.*;
import org.antlr.v4.runtime.tree.ParseTree;
import java.util.*;
public class Main {
public static void main(String[] args) {
/*
// Expression.g4
grammar Expression;
expression
: '-' expression
| expression ('*' | '/') expression
| expression ('+' | '-') expression
| '(' expression ')'
| NUMBER
| VARIABLE
;
NUMBER
: [0-9]+ ( '.' [0-9]+ )?
;
VARIABLE
: [a-zA-Z] [a-zA-Z0-9]+
;
SPACE
: [ \t\r\n] -> skip
;
*/
String source = "3 + 42 * (PI - 3.14159)";
ExpressionLexer lexer = new ExpressionLexer(CharStreams.fromString(source));
ExpressionParser parser = new ExpressionParser(new CommonTokenStream(lexer));
SimpleTree tree = new SimpleTree.Builder()
.withParser(parser)
.withParseTree(parser.expression())
.withDisplaySymbolicName(false)
.build();
DotOptions options = new DotOptions.Builder()
.withParameters(" labelloc=\"t\";\n label=\"Expression Tree\";\n\n")
.withLexerRuleShape("circle")
.build();
System.out.println(new DotTreeRepresentation().display(tree, options));
}
}
class DotTreeRepresentation {
public String display(SimpleTree tree) {
return display(tree, DotOptions.DEFAULT);
}
public String display(SimpleTree tree, DotOptions options) {
return display(new InOrderTraversal().traverse(tree), options);
}
public String display(List<SimpleTree.Node> nodes, DotOptions options) {
StringBuilder builder = new StringBuilder("graph tree {\n\n");
Map<SimpleTree.Node, String> nodeNameMap = new HashMap<>();
int nodeCount = 0;
if (options.parameters != null) {
builder.append(options.parameters);
}
for (SimpleTree.Node node : nodes) {
nodeCount++;
String nodeName = String.format("node_%s", nodeCount);
nodeNameMap.put(node, nodeName);
builder.append(String.format(" %s [label=\"%s\", shape=%s];\n",
nodeName,
node.getLabel().replace("\"", "\\\""),
node.isTokenNode() ? options.lexerRuleShape : options.parserRuleShape));
}
builder.append("\n");
for (SimpleTree.Node node : nodes) {
String name = nodeNameMap.get(node);
for (SimpleTree.Node child : node.getChildren()) {
String childName = nodeNameMap.get(child);
builder.append(" ").append(name).append(" -- ").append(childName).append("\n");
}
}
return builder.append("}\n").toString();
}
}
class InOrderTraversal {
public List<SimpleTree.Node> traverse(SimpleTree tree) {
if (tree == null)
throw new IllegalArgumentException("tree == null");
List<SimpleTree.Node> nodes = new ArrayList<>();
traverse(tree.root, nodes);
return nodes;
}
private void traverse(SimpleTree.Node node, List<SimpleTree.Node> nodes) {
if (node.hasChildren()) {
traverse(node.getChildren().get(0), nodes);
}
nodes.add(node);
for (int i = 1; i < node.getChildCount(); i++) {
traverse(node.getChild(i), nodes);
}
}
}
class DotOptions {
public static final DotOptions DEFAULT = new DotOptions.Builder().build();
public static final String DEFAULT_PARAMETERS = null;
public static final String DEFAULT_LEXER_RULE_SHAPE = "box";
public static final String DEFAULT_PARSER_RULE_SHAPE = "ellipse";
public static class Builder {
private String parameters = DEFAULT_PARAMETERS;
private String lexerRuleShape = DEFAULT_LEXER_RULE_SHAPE;
private String parserRuleShape = DEFAULT_PARSER_RULE_SHAPE;
public DotOptions.Builder withParameters(String parameters) {
this.parameters = parameters;
return this;
}
public DotOptions.Builder withLexerRuleShape(String lexerRuleShape) {
this.lexerRuleShape = lexerRuleShape;
return this;
}
public DotOptions.Builder withParserRuleShape(String parserRuleShape) {
this.parserRuleShape = parserRuleShape;
return this;
}
public DotOptions build() {
if (lexerRuleShape == null)
throw new IllegalStateException("lexerRuleShape == null");
if (parserRuleShape == null)
throw new IllegalStateException("parserRuleShape == null");
return new DotOptions(parameters, lexerRuleShape, parserRuleShape);
}
}
public final String parameters;
public final String lexerRuleShape;
public final String parserRuleShape;
private DotOptions(String parameters, String lexerRuleShape, String parserRuleShape) {
this.parameters = parameters;
this.lexerRuleShape = lexerRuleShape;
this.parserRuleShape = parserRuleShape;
}
}
class SimpleTree {
public static class Builder {
private Parser parser = null;
private ParseTree parseTree = null;
private Set<Integer> ignoredTokenTypes = new HashSet<>();
private boolean displaySymbolicName = true;
public SimpleTree build() {
if (parser == null) {
throw new IllegalStateException("parser == null");
}
if (parseTree == null) {
throw new IllegalStateException("parseTree == null");
}
return new SimpleTree(parser, parseTree, ignoredTokenTypes, displaySymbolicName);
}
public SimpleTree.Builder withParser(Parser parser) {
this.parser = parser;
return this;
}
public SimpleTree.Builder withParseTree(ParseTree parseTree) {
this.parseTree = parseTree;
return this;
}
public SimpleTree.Builder withIgnoredTokenTypes(Integer... ignoredTokenTypes) {
this.ignoredTokenTypes = new HashSet<>(Arrays.asList(ignoredTokenTypes));
return this;
}
public SimpleTree.Builder withDisplaySymbolicName(boolean displaySymbolicName) {
this.displaySymbolicName = displaySymbolicName;
return this;
}
}
public final SimpleTree.Node root;
private SimpleTree(Parser parser, ParseTree parseTree, Set<Integer> ignoredTokenTypes, boolean displaySymbolicName) {
this.root = new SimpleTree.Node(parser, parseTree, ignoredTokenTypes, displaySymbolicName);
}
public SimpleTree(SimpleTree.Node root) {
if (root == null)
throw new IllegalArgumentException("root == null");
this.root = root;
}
public SimpleTree copy() {
return new SimpleTree(root.copy());
}
public String toLispTree() {
StringBuilder builder = new StringBuilder();
toLispTree(this.root, builder);
return builder.toString().trim();
}
private void toLispTree(SimpleTree.Node node, StringBuilder builder) {
if (node.isLeaf()) {
builder.append(node.getLabel()).append(" ");
}
else {
builder.append("(").append(node.label).append(" ");
for (SimpleTree.Node child : node.children) {
toLispTree(child, builder);
}
builder.append(") ");
}
}
#Override
public String toString() {
return String.format("%s", this.root);
}
public static class Node {
protected String label;
protected int level;
protected boolean isTokenNode;
protected List<SimpleTree.Node> children;
Node(Parser parser, ParseTree parseTree, Set<Integer> ignoredTokenTypes, boolean displaySymbolicName) {
this(parser.getRuleNames()[((RuleContext)parseTree).getRuleIndex()], 0, false);
traverse(parseTree, this, parser, ignoredTokenTypes, displaySymbolicName);
}
public Node(String label, int level, boolean isTokenNode) {
this.label = label;
this.level = level;
this.isTokenNode = isTokenNode;
this.children = new ArrayList<>();
}
public void replaceWith(SimpleTree.Node node) {
this.label = node.label;
this.level = node.level;
this.isTokenNode = node.isTokenNode;
this.children.remove(node);
this.children.addAll(node.children);
}
public SimpleTree.Node copy() {
SimpleTree.Node copy = new SimpleTree.Node(this.label, this.level, this.isTokenNode);
for (SimpleTree.Node child : this.children) {
copy.children.add(child.copy());
}
return copy;
}
public void normalizeLevels(int level) {
this.level = level;
for (SimpleTree.Node child : children) {
child.normalizeLevels(level + 1);
}
}
public boolean hasChildren() {
return !children.isEmpty();
}
public boolean isLeaf() {
return !hasChildren();
}
public int getChildCount() {
return children.size();
}
public SimpleTree.Node getChild(int index) {
return children.get(index);
}
public int getLevel() {
return level;
}
public String getLabel() {
return label;
}
public boolean isTokenNode() {
return isTokenNode;
}
public List<SimpleTree.Node> getChildren() {
return new ArrayList<>(children);
}
private void traverse(ParseTree parseTree, SimpleTree.Node parent, Parser parser, Set<Integer> ignoredTokenTypes, boolean displaySymbolicName) {
List<SimpleTree.ParseTreeParent> todo = new ArrayList<>();
for (int i = 0; i < parseTree.getChildCount(); i++) {
ParseTree child = parseTree.getChild(i);
if (child.getPayload() instanceof CommonToken) {
CommonToken token = (CommonToken) child.getPayload();
if (!ignoredTokenTypes.contains(token.getType())) {
String tempText = displaySymbolicName ?
String.format("%s: '%s'",
parser.getVocabulary().getSymbolicName(token.getType()),
token.getText()
.replace("\r", "\\r")
.replace("\n", "\\n")
.replace("\t", "\\t")
.replace("'", "\\'")) :
String.format("%s",
token.getText()
.replace("\r", "\\r")
.replace("\n", "\\n")
.replace("\t", "\\t"));
if (parent.label == null) {
parent.label = tempText;
}
else {
parent.children.add(new SimpleTree.Node(tempText, parent.level + 1, true));
}
}
}
else {
SimpleTree.Node node = new SimpleTree.Node(parser.getRuleNames()[((RuleContext)child).getRuleIndex()], parent.level + 1, false);
parent.children.add(node);
todo.add(new SimpleTree.ParseTreeParent(child, node));
}
}
for (SimpleTree.ParseTreeParent wrapper : todo) {
traverse(wrapper.parseTree, wrapper.parent, parser, ignoredTokenTypes, displaySymbolicName);
}
}
#Override
public String toString() {
return String.format("{label=%s, level=%s, isTokenNode=%s, children=%s}", label, level, isTokenNode, children);
}
}
private static class ParseTreeParent {
final ParseTree parseTree;
final SimpleTree.Node parent;
private ParseTreeParent(ParseTree parseTree, SimpleTree.Node parent) {
this.parseTree = parseTree;
this.parent = parent;
}
}
}
And if you paste the output in a DOT viewer, you will get this:

You may also want to look at alternatives. DOT graphs aren't the pretties among possible graph representations. Maybe you like an svg graph instead? If so have a look at the ANTLR4 grammar extension for Visual Studio Code, which generates and exports an svg graphic with the click of a mouse button (and you can style that with own CSS code).

Is it possible to flatten the JSON hierarchy with Gson?

I use Gson to convert JSON data to Java objects. However, the JSON structure has an extra field which could be flattened. Is this possible to do with Gson?
To elaborate (since this is rather difficult to explain), the JSON looks something like this:
{
"foo": "bar",
"data": {
"first": 0,
"second": 1,
"third": 2
}
}
This produces two classes, one for the parent and one for data, like this:
public class Entry {
private String foo;
private Data data;
}
public class Data {
private int first;
private int second;
private int third;
}
I'd like to "flatten" the data field into the parent object so that the Java class would look something like this:
public class Entry {
private String foo;
private int first;
private int second;
private int third;
}
Is this possible with Gson, using e.g. TypeAdapters?

I'll show you demo and you decide for yourself do you really want this... Because it makes TypeAdapter code hard to read.
private static class EntryTypeAdapter extends TypeAdapter<Entry> {
// without registerTypeAdapter(Entry.class, new EntryTypeAdapter())
private Gson gson = new GsonBuilder()
// ignore "foo" from deserialization and serialization
.setExclusionStrategies(new TestExclStrat()).create();
#Override
public void write(JsonWriter out, Entry value) throws IOException {
out.beginObject();
out.name("foo");
out.value(value.foo);
out.name("data");
out.value(gson.toJson(value));
out.endObject();
}
#Override
public Entry read(JsonReader in) throws IOException {
Entry entry = null;
String foo = null;
in.beginObject();
while (in.hasNext()) {
String name = in.nextName();
if (name.equals("foo")) {
foo = in.nextString();
} else if (name.equals("data")) {
entry = gson.fromJson(in, Entry.class);
} else {
in.skipValue();
}
}
in.endObject();
if(entry!= null) entry.foo = foo;
return entry;
}
public class TestExclStrat implements ExclusionStrategy {
public boolean shouldSkipClass(Class<?> arg0) {
return false;
}
public boolean shouldSkipField(FieldAttributes f) {
return f.getName().equals("foo");
}
}
}
Can test it with this:
public static void main(String[] args) throws IOException, JSONException {
String jsonString = "{\n" +
" \"foo\": \"bar\",\n" +
" \"data\": {\n" +
" \"first\": 0,\n" +
" \"second\": 1,\n" +
" \"third\": 2\n" +
" }\n" +
"}";
Gson gson = new GsonBuilder()
.registerTypeAdapter(Entry.class, new EntryTypeAdapter()).create();
Entry el = gson.fromJson(jsonString, Entry.class);
String serialized = gson.toJson(el);
System.out.println(serialized);
}
public static class Entry {
public String foo;
public Integer first;
public Integer second;
public Integer third;
}
You could also do something like this:
// even more complicated version without inner Gson help
public Entry readOption2(JsonReader in) throws IOException {
Entry entry = new Entry();
in.beginObject();
while (in.hasNext()) {
String name = in.nextName();
if (name.equals("foo")) {
entry.foo = in.nextString();
} else if (name.equals("data")) {
in.beginObject();
while (in.hasNext()) {
name = in.nextName();
if (name.equals("first")) {
entry.first = in.nextInt();
} else if (name.equals("second")) {
entry.second = in.nextInt();
} else if (name.equals("third")) {
entry.third = in.nextInt();
}else{
in.skipValue();
}
}
in.endObject();
} else {
in.skipValue();
}
}
in.endObject();
return entry;
}