ANTLR: help needed with custom interpreter - antlr

I have some problems with "gathering" the data. It seems my code knows how to visit each tree and calculate values correctly but something is missing as the result is always null. Help is appreciated.
.g4 file:
grammar MyParser;
ID: [a-z]+ ;
Operator_I: '&';
Operator_U: '|';
EQ: '=';
expr: '(' expr ')' #wParExpr
| expr op=(Operator_I | Operator_U) expr #wExpr
| ID wID
;
corr: 'C=' (expr)* #Result;
.java code
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.tree.TerminalNode;
import java.util.*;
public class EvalVisitor extends MyParserBaseVisitor<Value> {
#Override
public Value visitWParExpr(MyParserParser.WParExprContext ctx) {
// TODO Auto-generated method stub
System.out.println("visitWParExpr:"+ctx.getText());
return visitChildren(ctx);
}
#Override
public Value visitWID(MyParserParser.WIDContext ctx) {
// TODO Auto-generated method stub
System.out.println("id:"+ctx.getText());
String id=ctx.getText();
Value v = getMemory().get(id);
l("ret id:"+v.value);
return v;
}
#Override
public Value visitWExpr(MyParserParser.WExprContext ctx) {
String left=ctx.expr(0).getText();
String right=ctx.expr(1).getText();
if (ctx.getText().contains("(")) {
return visitChildren(ctx);
}
l("visitWExpr, left="+left+",right:"+right);
String op=ctx.op.getText();
Value lefti = visit(ctx.expr(0));
Value righti = visit(ctx.expr(1));;
Value v= null;
if (op.equals("&")) {
v=new Value(lefti.asDouble()+righti.asDouble());
} else if (op.equals("|")) {
v=new Value(lefti.asDouble()+righti.asDouble());
}
l("return:"+v.asString());
return v;
}
public void l(String log) {
System.out.println(log);
}
#Override public Value visitResult(MyParserParser.ResultContext ctx) {
String s1 = ctx.getText();
String s2 = ctx.expr(0).getText();
l("VisitResult:"+s1+", expr="+s2);
Value v = visit(ctx.expr(0));
l("VisitResult end:"+v);
memory.put("result",v);
return v;
}
private Map<String, Value> memory = new HashMap<String, Value>();
public EvalVisitor() {
getMemory().put("a", new Value(new Double(5)));
getMemory().put("b", new Value(new Double(9)));
getMemory().put("c", new Value(new Double(12)));
}
public Map<String, Value> getMemory() {
return memory;
}
public void setValues(Map<String, Value> values) {
this.memory = values;
}
}
public static void main(String[] args) throws Exception {
MyParserLexer lexer = new MyParserLexer(new ANTLRFileStream("c:\\test.mu"));
MyParserParser parser = new MyParserParser(new CommonTokenStream(lexer));
ParseTree tree = parser.corr();
EvalVisitor visitor = new EvalVisitor();
visitor.visit(tree);
System.out.println(visitor.getMemory());
}
output:
VisitResult:C=(a&b)&(b&c)|(c&c), expr=(a&b)&(b&c)|(c&c)
visitWParExpr:(a&b)
visitWExpr, left=a,right:b
id:a
ret id:5.0
id:b
ret id:9.0
return:14.0
visitWParExpr:(b&c)
visitWExpr, left=b,right:c
id:b
ret id:9.0
id:c
ret id:12.0
return:21.0
visitWParExpr:(c&c)
visitWExpr, left=c,right:c
id:c
ret id:12.0
id:c
ret id:12.0
return:24.0
VisitResult end:null
{result=null, a=5.0, b=9.0, c=12.0}
EDIT
I think I got it working. The reason was that I did not do evaluations in wExpr context. I didn't understand that I need to evaluate subtrees (if context contains "(") there unless I have only id available which resolves down to double value.
Can this same be achieved in more efficient manner or implement the same functionality some other way ? For example is it possible to do this if I had only one expr() function that gets all the expressions ?
Here's revised code if someone is interested:
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.runtime.tree.TerminalNode;
import java.util.*;
public class EvalVisitor extends MyParserBaseVisitor<Value> {
#Override
public Value visitWParExpr(MyParserParser.WParExprContext ctx) {
// TODO Auto-generated method stub
System.out.println("visitWParExpr:"+ctx.getText()+", expr:"+ctx.expr().getText());
String expr= ctx.expr().getText();
Value v = visit(ctx.expr());
l("WParExpr:"+v);
memory.put(expr, v);
return v;
}
#Override
public Value visitWID(MyParserParser.WIDContext ctx) {
// TODO Auto-generated method stub
System.out.println("VisitWID:"+ctx.getText());
String id=ctx.getText();
Value v = getMemory().get(id);
l("ret id:"+v.value);
return v;
}
#Override
public Value visitWExpr(MyParserParser.WExprContext ctx) {
String left=ctx.expr(0).getText();
String right=ctx.expr(1).getText();
l("visitWExpr:"+ctx.getText()+", left="+left+",right:"+right);
Value v=null;
Value lv=null;
Value rv=null;
if (ctx.getText().contains("(")) {
lv = visit(ctx.expr(0));
rv = visit(ctx.expr(1));
} else {
lv = visit(ctx.expr(0));
rv = visit(ctx.expr(1));;
}
String op=ctx.op.getText();
l("lv="+lv+",rv="+rv+",op="+op);
if (op.equals("&")) {
v=new Value(lv.asDouble()+rv.asDouble());
} else if (op.equals("|")) {
v=new Value(lv.asDouble()*rv.asDouble());
}
l("return:"+v.asString());
return v;
}
public void l(String log) {
System.out.println(log);
}
#Override public Value visitResult(MyParserParser.ResultContext ctx) {
String s1 = ctx.getText();
String s2 = ctx.expr().getText();
l("VisitResult:"+s1+", expr="+s2);
Value v = visit(ctx.expr());
l("VisitResult end:"+v);
memory.put("result",v);
return v;
}
private Map<String, Value> memory = new HashMap<String, Value>();
public EvalVisitor() {
getMemory().put("a", new Value(new Double(5)));
getMemory().put("b", new Value(new Double(9)));
getMemory().put("c", new Value(new Double(12)));
}
public Map<String, Value> getMemory() {
return memory;
}
public void setValues(Map<String, Value> values) {
this.memory = values;
}
}
Sample output for: string:
C=((a&b)&(b&c))|(c&c)|((b&a)&(c&b))|(c|c)
VisitResult:C=((a&b)&(b&c))|(c&c)|((b&a)&(c&b))|(c|c), expr=((a&b)&(b&c))|(c&c)|((b&a)&(c&b))|(c|c)
visitWExpr:((a&b)&(b&c))|(c&c)|((b&a)&(c&b))|(c|c), left=((a&b)&(b&c))|(c&c)|((b&a)&(c&b)),right:(c|c)
visitWExpr:((a&b)&(b&c))|(c&c)|((b&a)&(c&b)), left=((a&b)&(b&c))|(c&c),right:((b&a)&(c&b))
visitWExpr:((a&b)&(b&c))|(c&c), left=((a&b)&(b&c)),right:(c&c)
visitWParExpr:((a&b)&(b&c)), expr:(a&b)&(b&c)
visitWExpr:(a&b)&(b&c), left=(a&b),right:(b&c)
visitWParExpr:(a&b), expr:a&b
visitWExpr:a&b, left=a,right:b
VisitWID:a
ret id:5.0
VisitWID:b
ret id:9.0
lv=5.0,rv=9.0,op=&
return:14.0
WParExpr:14.0
visitWParExpr:(b&c), expr:b&c
visitWExpr:b&c, left=b,right:c
VisitWID:b
ret id:9.0
VisitWID:c
ret id:12.0
lv=9.0,rv=12.0,op=&
return:21.0
WParExpr:21.0
lv=14.0,rv=21.0,op=&
return:35.0
WParExpr:35.0
visitWParExpr:(c&c), expr:c&c
visitWExpr:c&c, left=c,right:c
VisitWID:c
ret id:12.0
VisitWID:c
ret id:12.0
lv=12.0,rv=12.0,op=&
return:24.0
WParExpr:24.0
lv=35.0,rv=24.0,op=|
return:840.0
visitWParExpr:((b&a)&(c&b)), expr:(b&a)&(c&b)
visitWExpr:(b&a)&(c&b), left=(b&a),right:(c&b)
visitWParExpr:(b&a), expr:b&a
visitWExpr:b&a, left=b,right:a
VisitWID:b
ret id:9.0
VisitWID:a
ret id:5.0
lv=9.0,rv=5.0,op=&
return:14.0
WParExpr:14.0
visitWParExpr:(c&b), expr:c&b
visitWExpr:c&b, left=c,right:b
VisitWID:c
ret id:12.0
VisitWID:b
ret id:9.0
lv=12.0,rv=9.0,op=&
return:21.0
WParExpr:21.0
lv=14.0,rv=21.0,op=&
return:35.0
WParExpr:35.0
lv=840.0,rv=35.0,op=|
return:29400.0
visitWParExpr:(c|c), expr:c|c
visitWExpr:c|c, left=c,right:c
VisitWID:c
ret id:12.0
VisitWID:c
ret id:12.0
lv=12.0,rv=12.0,op=|
return:144.0
WParExpr:144.0
lv=29400.0,rv=144.0,op=|
return:4233600.0
VisitResult end:4233600.0
{result=4233600.0, a=5.0, c&c=24.0, b=9.0, c=12.0, (a&b)&(b&c)=35.0, c|c=144.0, a&b=14.0, b&a=14.0, (b&a)&(c&b)=35.0, b&c=21.0, c&b=21.0}

Related

APACH POI parsing HTML with JSOUP

Parsing an HTML text constaining <font> tags does not reset the size and family font after </font>
My code works pretty well except after </font>.
Before <font size=9> blablabla.. the text size was 11. I was expecting that after </font> the text size was reset to 11, but it still remains at 9. Same thing for the font family.
Certainly I have misunderstood how to use jsoup. I'd better use CSS, but I don't know how to do.
Thanks for help.
package test;
import java.awt.Color;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.math.BigInteger;
import org.apache.poi.xwpf.usermodel.UnderlinePatterns;
import org.apache.poi.xwpf.usermodel.VerticalAlign;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.apache.poi.xwpf.usermodel.XWPFParagraph;
import org.apache.poi.xwpf.usermodel.XWPFRun;
import org.apache.poi.xwpf.usermodel.XWPFTable;
import org.apache.poi.xwpf.usermodel.XWPFTableCell;
import org.apache.poi.xwpf.usermodel.XWPFTableRow;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Node;
import org.jsoup.nodes.TextNode;
import org.jsoup.select.Elements;
import org.jsoup.select.NodeTraversor;
import org.jsoup.select.NodeVisitor;
import org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTblLayoutType;
import org.openxmlformats.schemas.wordprocessingml.x2006.main.STTblLayoutType;
public class ReadHtml
{
protected static java.util.Vector<String> contenu = null;
org.apache.xmlbeans.XmlCursor cursor = null;
class WStyle
{
protected String police = "Times New Roman";
protected int taille = 11;
protected Color couleur = Color.black;
protected boolean gras = false;
protected boolean italique = false;
public WStyle() // constructeur
{
super();
}
protected String getPolice() {return police;}
protected int getTaille() {return taille;}
protected Color getCouleur() {return couleur;}
protected boolean getGras() {return gras;}
protected boolean getItalique() {return italique;}
protected void setPolice(String p) {police=p;}
protected void setTaille(int t) {taille=t;}
protected void setCouleur(Color c) {couleur=c;}
protected void setGras(boolean g) {gras=g;}
protected void setItalique(boolean i) {italique=i;}
}
public ReadHtml()
{
super();
contenu = new java.util.Vector<String>();
createWordFile();
}
private XWPFParagraph getTableParagraph(XWPFTableCell cell, String html)
{
cell.removeParagraph(0);
XWPFParagraph paragraph = cell.addParagraph();
paragraph.setSpacingAfterLines(0);
paragraph.setSpacingAfter(0);
Document htmlDocument = Jsoup.parse(html);
Elements htmlParagraphs = htmlDocument.select("p");
for(Element htmlParagraph : htmlParagraphs)
{
System.out.println(htmlParagraph);
ParagraphNodeVisitor nodeVisitor = new ParagraphNodeVisitor(paragraph);
NodeTraversor.traverse(nodeVisitor, htmlParagraph);
}
return paragraph;
}
private void createWordFile()
{
XWPFParagraph para = null;
try
{
XWPFDocument document = new XWPFDocument();
FileOutputStream out = new FileOutputStream(new File("./", "NewTable.docx"));
XWPFTable table = document.createTable();
CTTblLayoutType type = table.getCTTbl().getTblPr().addNewTblLayout();
type.setType(STTblLayoutType.FIXED);
table.getCTTbl().addNewTblGrid().addNewGridCol().setW(BigInteger.valueOf(1670));
table.getCTTbl().getTblGrid().addNewGridCol().setW(BigInteger.valueOf(6000));
String myTexte = "<html><head</head><body><p><font face=\"Verdana\" size=11>Good Morning</font> <font size=9 face=\"Times\"> " +
"<i><b>how are you today </b></i></font> Not so bad.<br>Thanks";
// first line
XWPFTableRow tableRow= table.getRow(0);
para = getTableParagraph(tableRow.getCell(0), "<p>Row #1, Col. #1");
tableRow .getCell(0).setParagraph(para);
XWPFTableCell cell = tableRow.createCell();
para = getTableParagraph(cell, myTexte); // Row #1, Col. #2
tableRow .getCell(1).setParagraph(para);
// seconde line
tableRow= table.createRow();
para = getTableParagraph(tableRow.getCell(0), "<p>Row #2, Col. #1");
tableRow .getCell(0).setParagraph(para);
para = getTableParagraph(tableRow.getCell(1), "<p>Row #2, Col. #2");
tableRow.getCell(1).setParagraph(para);
document.write(out);
document.close();
out.close();
System.out.println("NewTable.docx written successully");
}
catch (FileNotFoundException e) {System.out.println("File exception --> " + e.toString()); }
catch (IOException e) {System.out.println("I/O exception --> " + e.toString()); }
catch (Exception e) {System.out.println("Other exception --> " + e.toString()); }
}
public class ParagraphNodeVisitor implements NodeVisitor
{
String nodeName;
String fontFace;
String fontType;
boolean needNewRun;
boolean isItalic;
boolean isBold;
boolean isUnderlined;
int fontSize;
String fontColor;
VerticalAlign align = VerticalAlign.BASELINE ;
XWPFParagraph paragraph;
XWPFRun run;
ParagraphNodeVisitor(XWPFParagraph paragraph)
{
this.paragraph = paragraph;
this.run = paragraph.createRun();
this.nodeName = "";
this.needNewRun = false;
this.isItalic = false;
this.isBold = false;
this.isUnderlined = false;
this.fontSize = 11;
this.fontColor = "000000";
this.fontFace="Times";
}
#Override
public void head(Node node, int depth)
{
nodeName = node.nodeName();
needNewRun = false;
if ("#text".equals(nodeName))
{
run.setText(((TextNode)node).text());
needNewRun = true; //after setting the text in the run a new run is needed
}
else if ("i".equals(nodeName)) {isItalic = true;}
else if ("b".equals(nodeName)) {isBold = true;}
else if ("sup".equals(nodeName)){align = VerticalAlign.SUPERSCRIPT ;}
else if ("u".equals(nodeName)) {isUnderlined = true;}
else if ("br".equals(nodeName)) {run.addBreak();}
else if ("p".equals(nodeName)) {run.addBreak();}
else if ("font".equals(nodeName))
{
fontColor = (!"".equals(node.attr("color")))?node.attr("color").substring(1):"000000";
fontSize = (!"".equals(node.attr("size")))?Integer.parseInt(node.attr("size")):11;
fontFace = (!"".equals(node.attr("face")))?node.attr("face"):"Times";
}
if (needNewRun) run = paragraph.createRun();
needNewRun = false;
run.setItalic(isItalic);
run.setBold(isBold);
if (isUnderlined) run.setUnderline(UnderlinePatterns.SINGLE);
else run.setUnderline(UnderlinePatterns.NONE);
run.setColor(fontColor);
run.setFontSize(fontSize);
run.setFontFamily(fontFace);
run.setSubscript(align);
}
#Override
public void tail(Node node, int depth)
{
nodeName = node.nodeName();
System.out.println("Node=" + nodeName);
if ("i".equals(nodeName)) {isItalic = false;}
else if ("b".equals(nodeName)) {isBold = false;}
else if ("u".equals(nodeName)) {isUnderlined = false;}
else if ("sup".equals(nodeName)) {align= VerticalAlign.BASELINE ;}
else if ("font".equals("nodeName"))
{
fontColor = "000000";
fontSize = 11;
fontFace="Times";
System.out.println("Family=" + fontFace + " Taille=" + fontSize);
}
if (needNewRun) run = paragraph.createRun();
needNewRun = false;
run.setItalic(isItalic);
run.setBold(isBold);
if (isUnderlined) run.setUnderline(UnderlinePatterns.SINGLE); else run.setUnderline(UnderlinePatterns.NONE);
run.setColor(fontColor);
run.setFontSize(fontSize);
run.setFontFamily(fontFace);
run.setSubscript(align);
}
}
public static void main(String[] args)
{
new ReadHtml() ;
}
}
Please change the following line in your tail method, from
else if ("font".equals("nodeName"))
to
else if ("font".equals(nodeName))
You've compared two string literals instead of compare a string literal to the variable. Because of the typo the condition is always false, therefore fontSize is never reseted.

error is occuring while creating custom tokenizer in lucene 7.3

I m trying to create new tokenizer by refering book tamingtext (which uses lucene 3.+ api) using new lucene api 7.3, but it is giving me error as mentioned below
java.lang.IllegalStateException: TokenStream contract violation: reset()/close() call missing, reset() called multiple times, or subclass does not call super.reset(). Please see Javadocs of TokenStream class for more information about the correct consuming workflow.
at org.apache.lucene.analysis.Tokenizer$1.read(Tokenizer.java:109)
at java.io.Reader.read(Reader.java:140)
at solr.SentenceTokenizer.fillSentences(SentenceTokenizer.java:43)
at solr.SentenceTokenizer.incrementToken(SentenceTokenizer.java:55)
at solr.NameFilter.fillSpans(NameFilter.java:56)
at solr.NameFilter.incrementToken(NameFilter.java:88)
at spec.solr.NameFilterTest.testNameFilter(NameFilterTest.java:81)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
Here is my SentenceTokenizer class
Initializing method, in older api there was super(Reader); but in current api there is no access to Reader class
public SentenceTokenizer(SentenceDetector detector) {
super();
setReader(reader);
this.sentenceDetector = detector;
}
Here is my reset method
#Override
public void reset() throws IOException {
super.reset();
sentenceDetector = null;
}
When i tried to access this method from custom TokenFilter, I m getting above error
public void fillSentences() throws IOException {
char[] c = new char[256];
int size = 0;
StringBuilder stringBuilder = new StringBuilder();
while ((size = input.read(c)) >= 0) {
stringBuilder.append(c, 0, size);
}
String temp = stringBuilder.toString();
inputSentence = temp.toCharArray();
sentenceSpans = sentenceDetector.sentPosDetect(temp);
tokenOffset = 0;
}
#Override
public final boolean incrementToken() throws IOException {
if (sentenceSpans == null) {
//invoking following method
fillSentences();
}
if (tokenOffset == sentenceSpans.length) {
return false;
}
Span sentenceSpan = sentenceSpans[tokenOffset];
clearAttributes();
int start = sentenceSpan.getStart();
int end = sentenceSpan.getEnd();
charTermAttribute.copyBuffer(inputSentence, start, end - start);
positionIncrementAttribute.setPositionIncrement(1);
offsetAttribute.setOffset(start, end);
tokenOffset++;
return true;
}
Here is my custom TokenFilter class
public final class NameFilter extends TokenFilter {
public static final String NE_PREFIX = "NE_";
private final Tokenizer tokenizer;
private final String[] tokenTypeNames;
private final NameFinderME[] nameFinderME;
private final KeywordAttribute keywordAttribute = addAttribute(KeywordAttribute.class);
private final PositionIncrementAttribute positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class);
private final CharTermAttribute charTermAttribute = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class);
private String text;
private int baseOffset;
private Span[] spans;
private String[] tokens;
private Span[][] foundNames;
private boolean[][] tokenTypes;
private int spanOffsets = 0;
private final Queue<AttributeSource.State> tokenQueue =
new LinkedList<>();
public NameFilter(TokenStream in, String[] modelNames, NameFinderME[] nameFinderME) {
super(in);
this.tokenizer = SimpleTokenizer.INSTANCE;
this.nameFinderME = nameFinderME;
this.tokenTypeNames = new String[modelNames.length];
for (int i = 0; i < modelNames.length; i++) {
this.tokenTypeNames[i] = NE_PREFIX + modelNames[i];
}
}
//consumes tokens from the upstream tokenizer and buffer them in a
//StringBuilder whose contents will be passed to opennlp
protected boolean fillSpans() throws IOException {
if (!this.input.incrementToken()) return false;
//process the next sentence from the upstream tokenizer
this.text = input.getAttribute(CharTermAttribute.class).toString();
this.baseOffset = this.input.getAttribute(OffsetAttribute.class).startOffset();
this.spans = this.tokenizer.tokenizePos(text);
this.tokens = Span.spansToStrings(spans, text);
this.foundNames = new Span[this.nameFinderME.length][];
for (int i = 0; i < nameFinderME.length; i++) {
this.foundNames[i] = nameFinderME[i].find(tokens);
}
//insize
this.tokenTypes = new boolean[this.tokens.length][this.nameFinderME.length];
for (int i = 0; i < nameFinderME.length; i++) {
Span[] spans = foundNames[i];
for (int j = 0; j < spans.length; j++) {
int start = spans[j].getStart();
int end = spans[j].getEnd();
for (int k = start; k < end; k++) {
this.tokenTypes[k][i] = true;
}
}
}
spanOffsets = 0;
return true;
}
#Override
public boolean incrementToken() throws IOException {
//if there's nothing in the queue
if(tokenQueue.peek()==null){
//no span or spans consumed
if (spans==null||spanOffsets>=spans.length){
if (!fillSpans())return false;
}
if (spanOffsets>=spans.length)return false;
//copy the token and any types
clearAttributes();
keywordAttribute.setKeyword(false);
positionIncrementAttribute.setPositionIncrement(1);
int startOffset = baseOffset +spans[spanOffsets].getStart();
int endOffset = baseOffset+spans[spanOffsets].getEnd();
offsetAttribute.setOffset(startOffset,endOffset);
charTermAttribute.setEmpty()
.append(tokens[spanOffsets]);
//determine of the current token is of a named entity type, if so
//push the current state into the queue and add a token reflecting
// any matching entity types.
boolean [] types = tokenTypes[spanOffsets];
for (int i = 0; i < nameFinderME.length; i++) {
if (types[i]){
keywordAttribute.setKeyword(true);
positionIncrementAttribute.setPositionIncrement(0);
tokenQueue.add(captureState());
positionIncrementAttribute.setPositionIncrement(1);
charTermAttribute.setEmpty().append(tokenTypeNames[i]);
}
}
}
spanOffsets++;
return true;
}
#Override
public void close() throws IOException {
super.close();
reset();
}
#Override
public void reset() throws IOException {
super.reset();
this.spanOffsets = 0;
this.spans = null;
}
#Override
public void end() throws IOException {
super.end();
reset();
}
}
here is my test case for following class
#Test
public void testNameFilter() throws IOException {
Reader in = new StringReader(input);
Tokenizer tokenizer = new SentenceTokenizer( detector);
tokenizer.reset();
NameFilter nameFilter = new NameFilter(tokenizer, modelName, nameFinderMES);
nameFilter.reset();
CharTermAttribute charTermAttribute;
PositionIncrementAttribute positionIncrementAttribute;
OffsetAttribute offsetAttribute;
int pass = 0;
while (pass < 2) {
int pos = 0;
int lastStart = 0;
int lastEnd = 0;
//error occur on below invoke
while (nameFilter.incrementToken()) {
}
}
I have added following changes in my code and it work fine but i m now sure it is correct answer
public SentenceTokenizer(Reader reader,SentenceDetector sentenceDetector) {
super();
this.input =reader;
this.sentenceDetector = sentenceDetector;
}

JSON response Using Volley in a recycler view in Android app

I am trying to print a json response response in a recyclerview. my Recyclerview is like a expanded listView. And my json resone is like this:
{"Table1":
[
{"filedate":"Oct 26, 2016"},
{"filedate":"Oct 18, 2016"}
],
"Table2":
[{
"filedate":"Oct 18, 2016",
"file1":"12.txt"
},
{
"filedate":"Oct 26, 2016",
"file1":"acerinvoice.pdf"
}
]}
and I trying to print this json resonse using this code:
private void prepareListData() {
// Volley's json array request object
StringRequest stringRequest = new StringRequest(Request.Method.POST, REGISTER_URL,
new Response.Listener<String>() {
//I think problrm is here
#Override
public void onResponse(String response) {
try {
**JSONObject object = new JSONObject(response);
JSONArray jsonarray = object.getJSONArray("Table1");
JSONArray jsonarray1 = object.getJSONArray("Table2");
for (int i = 0; i < jsonarray.length(); i++) {
try {
JSONObject obj = jsonarray.getJSONObject(i);
String str = obj.optString("filedate").trim();
int lth = jsonarray1.length();
data.add(new ExpandableListAdapter.Item(ExpandableListAdapter.HEADER, str));
for (int j = 0; j < jsonarray1.length(); j++) {
try {
JSONObject obj1 = jsonarray1.getJSONObject(j);
String str1 = obj1.optString("filedate").trim();
data.add(new ExpandableListAdapter.Item(ExpandableListAdapter.CHILD, str1));
Toast.makeText(getApplicationContext(), str1, Toast.LENGTH_LONG).show();**
//if condition
} catch (JSONException e) {
// Log.e(TAG, "JSON Parsing error: " + e.getMessage());
}
}
// adding movie to movies array
} catch (JSONException e) {
Log.e("gdshfsjkg", "JSON Parsing error: " + e.getMessage());
}
}
Log.d("test", String.valueOf(data));
Toast.makeText(getApplicationContext(), (CharSequence) data, Toast.LENGTH_LONG).show();
} catch (JSONException e) {
e.printStackTrace();
}
// notifying list adapter about data changes
// so that it renders the list view with updated data
recyclerview.setAdapter(new ExpandableListAdapter(data));
}
}, new Response.ErrorListener() {
#Override
public void onErrorResponse(VolleyError error) {
}
}){
#Override
protected Map<String, String> getParams() {
Map<String, String> params = new HashMap<String, String>();
params.put(CLIENT, "4");
return params;
}
};
// Adding request to request queue
MyApplication.getInstance().addToRequestQueue(stringRequest);
}
ExpandableListAdapter
public class ExpandableListAdapter extends RecyclerView.Adapter<RecyclerView.ViewHolder> {
public static final int HEADER = 0;
public static final int CHILD = 1;
private List<Item> data;
public ExpandableListAdapter(List<Item> data) {
this.data = data;
}
#Override
public RecyclerView.ViewHolder onCreateViewHolder(ViewGroup parent, int type) {
View view = null;
Context context = parent.getContext();
float dp = context.getResources().getDisplayMetrics().density;
int subItemPaddingLeft = (int) (18 * dp);
int subItemPaddingTopAndBottom = (int) (5 * dp);
switch (type) {
case HEADER:
LayoutInflater inflater = (LayoutInflater) parent.getContext().getSystemService(Context.LAYOUT_INFLATER_SERVICE);
view = inflater.inflate(R.layout.list_header, parent, false);
ListHeaderViewHolder header = new ListHeaderViewHolder(view);
return header;
case CHILD:
TextView itemTextView = new TextView(context);
itemTextView.setPadding(subItemPaddingLeft, subItemPaddingTopAndBottom, 0, subItemPaddingTopAndBottom);
itemTextView.setTextColor(0x88000000);
itemTextView.setLayoutParams(
new ViewGroup.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.WRAP_CONTENT));
return new RecyclerView.ViewHolder(itemTextView) {
};
}
return null;
}
public void onBindViewHolder(RecyclerView.ViewHolder holder, int position) {
final Item item = data.get(position);
switch (item.type) {
case HEADER:
final ListHeaderViewHolder itemController = (ListHeaderViewHolder) holder;
itemController.refferalItem = item;
itemController.header_title.setText(item.text);
if (item.invisibleChildren == null) {
itemController.btn_expand_toggle.setImageResource(R.drawable.circle_minus);
} else {
itemController.btn_expand_toggle.setImageResource(R.drawable.circle_plus);
}
itemController.btn_expand_toggle.setOnClickListener(new View.OnClickListener() {
#Override
public void onClick(View v) {
if (item.invisibleChildren == null) {
item.invisibleChildren = new ArrayList<Item>();
int count = 0;
int pos = data.indexOf(itemController.refferalItem);
while (data.size() > pos + 1 && data.get(pos + 1).type == CHILD) {
item.invisibleChildren.add(data.remove(pos + 1));
count++;
}
notifyItemRangeRemoved(pos + 1, count);
itemController.btn_expand_toggle.setImageResource(R.drawable.circle_plus);
} else {
int pos = data.indexOf(itemController.refferalItem);
int index = pos + 1;
for (Item i : item.invisibleChildren) {
data.add(index, i);
index++;
}
notifyItemRangeInserted(pos + 1, index - pos - 1);
itemController.btn_expand_toggle.setImageResource(R.drawable.circle_minus);
item.invisibleChildren = null;
}
}
});
break;
case CHILD:
TextView itemTextView = (TextView) holder.itemView;
itemTextView.setText(data.get(position).text);
break;
}
}
#Override
public int getItemViewType(int position) {
return data.get(position).type;
}
#Override
public int getItemCount() {
return data.size();
}
private static class ListHeaderViewHolder extends RecyclerView.ViewHolder {
public TextView header_title;
public ImageView btn_expand_toggle;
public Item refferalItem;
public ListHeaderViewHolder(View itemView) {
super(itemView);
header_title = (TextView) itemView.findViewById(R.id.header_title);
btn_expand_toggle = (ImageView) itemView.findViewById(R.id.btn_expand_toggle);
}
}
public static class Item {
public int type;
public String text;
public List<Item> invisibleChildren;
public Item() {
}
public Item(int type, String text) {
this.type = type;
this.text = text;
}
}
}
But this code print nothing in my recycler UI.It show only a empty layout.I have also tried to print the response in my log and it prints whole response together.I want to print table1 contents in headers and table2 contents in as child(items) with their respective headers.how to do this?If any other information needed please ask..
You should probably check your response first of everything. As per your requirements you should use HashMap and ArrayList. Follow these following steps:
1.Take a ArrayLsit and store child data of 1st heading and so on with index starting from 0.
2.Store headers in HashMap as key.
ex: HashMap<String,ArrayList<Data>> hm;
hm.put("your first heading string","related arraylist");
Then use ExpandableListView to arrange parent and child items.
please check your json response this is invalid json response first try to validate your json formate after try to decode and add in recycler view.
Check Your Response into Logcat also the Exception. I've tested the Response you've supplied that contains some unnecessary "," that may cause the Exception.
I have solve my problem to just change above code like this..
private void prepareListData() {
// Volley's json array request object
StringRequest stringRequest = new StringRequest(Request.Method.POST, REGISTER_URL,
new Response.Listener<String>() {
#Override
public void onResponse(String response) {
JSONObject object = null;
try {
object = new JSONObject(response);
} catch (JSONException e) {
e.printStackTrace();
}
JSONArray jsonarray = null;
JSONArray jsonarray1 = null;
try {
jsonarray = object.getJSONArray("Table1");
jsonarray1 = object.getJSONArray("Table1");
} catch (JSONException e) {
e.printStackTrace();
}
for (int i = 0; i < jsonarray.length(); i++) {
try {
JSONObject obj = jsonarray.getJSONObject(i);
//
String str = obj.optString("filedate").trim();
Log.d("test", str);
data.add(new ExpandableListAdapter.Item(ExpandableListAdapter.HEADER, str));
// Toast.makeText(getApplicationContext(), lth, Toast.LENGTH_LONG).show();
for (int j = 0; j < jsonarray1.length(); j++) {
try {
JSONObject obj1 = jsonarray1.getJSONObject(j);
String str1 = obj1.optString("filedate").trim();
String str3 = obj1.optString("category").trim();
String str2 = obj1.optString("file1").trim();
String str4 = obj1.optString("4").trim();
Toast.makeText(getApplicationContext(), "server data respone", Toast.LENGTH_LONG).show();
Toast.makeText(getApplicationContext(), "test"+str1, Toast.LENGTH_LONG).show();
if (str == str1) {
// data.add(new ExpandableListAdapter.Item(ExpandableListAdapter.CHILD, str1));
data.add(new ExpandableListAdapter.Item(ExpandableListAdapter.CHILD, str3));
data.add(new ExpandableListAdapter.Item(ExpandableListAdapter.CHILD, str2));
}
Toast.makeText(getApplicationContext(), str1, Toast.LENGTH_LONG).show();
//if condition
} catch (JSONException e) {
// Log.e(TAG, "JSON Parsing error: " + e.getMessage());
}
}
// adding movie to movies array
} catch (JSONException e) {
Log.e("gdshfsjkg", "JSON Parsing error: " + e.getMessage());
}
}
Log.d("test", String.valueOf(data));
// notifying list adapter about data changes
// so that it renders the list view with updated data
// adapterheader.notifyDataSetChanged();
recyclerview.setAdapter(new ExpandableListAdapter(data));
}
}, new Response.ErrorListener() {
#Override
public void onErrorResponse(VolleyError error) {
// VolleyLog.d(TAG, "Error: " + error.getMessage());
// hidePDialog();
}
}){
#Override
protected Map<String, String> getParams() {
Map<String, String> params = new HashMap<String, String>();
params.put(CLIENT, "4");
return params;
}
};
// Adding request to request queue
MyApplication.getInstance().addToRequestQueue(stringRequest);
}

With Lucene 4.3.1, How to get all terms which occur in sub-range of all docs

Suppose a lucene index with fields : date, content.
I want to get all terms value and frequency of docs whose date is yesterday. date field is keyword field. content field is analyzed and indexed.
Pls help me with sample code.
My solution source is as follow ...
/**
*
*
* #param reader
* #param fromDateTime
* - yyyymmddhhmmss
* #param toDateTime
* - yyyymmddhhmmss
* #return
*/
static public String top10(IndexSearcher searcher, String fromDateTime,
String toDateTime) {
String top10Query = "";
try {
Query query = new TermRangeQuery("tweetDate", new BytesRef(
fromDateTime), new BytesRef(toDateTime), true, false);
final BitSet bits = new BitSet(searcher.getIndexReader().maxDoc());
searcher.search(query, new Collector() {
private int docBase;
#Override
public void setScorer(Scorer scorer) throws IOException {
}
#Override
public void setNextReader(AtomicReaderContext context)
throws IOException {
this.docBase = context.docBase;
}
#Override
public void collect(int doc) throws IOException {
bits.set(doc + docBase);
}
#Override
public boolean acceptsDocsOutOfOrder() {
return false;
}
});
//
Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_43,
EnglishStopWords.getEnglishStopWords());
//
HashMap<String, Long> wordFrequency = new HashMap<>();
for (int wx = 0; wx < bits.length(); ++wx) {
if (bits.get(wx)) {
Document wd = searcher.doc(wx);
//
TokenStream tokenStream = analyzer.tokenStream("temp",
new StringReader(wd.get("content")));
// OffsetAttribute offsetAttribute = tokenStream
// .addAttribute(OffsetAttribute.class);
CharTermAttribute charTermAttribute = tokenStream
.addAttribute(CharTermAttribute.class);
tokenStream.reset();
while (tokenStream.incrementToken()) {
// int startOffset = offsetAttribute.startOffset();
// int endOffset = offsetAttribute.endOffset();
String term = charTermAttribute.toString();
if (term.length() < 2)
continue;
Long wl;
if ((wl = wordFrequency.get(term)) == null)
wordFrequency.put(term, 1L);
else {
wl += 1;
wordFrequency.put(term, wl);
}
}
tokenStream.end();
tokenStream.close();
}
}
analyzer.close();
// sort
List<String> occurterm = new ArrayList<String>();
for (String ws : wordFrequency.keySet()) {
occurterm.add(String.format("%06d\t%s", wordFrequency.get(ws),
ws));
}
Collections.sort(occurterm, Collections.reverseOrder());
// make query string by top 10 words
int topCount = 10;
for (String ws : occurterm) {
if (topCount-- == 0)
break;
String[] tks = ws.split("\\t");
top10Query += tks[1] + " ";
}
top10Query.trim();
} catch (IOException e) {
e.printStackTrace();
} finally {
}
// return top10 word string
return top10Query;
}

What's wrong with this Lucene TokenFilter?

Disclaimer: I've been coding for 36 of the last 41 hours. I have a headache. And I can't figure out why this combining TokenFilter is returning 2 tokens, both the first token from the source stream.
public class TokenCombiner extends TokenFilter {
/*
* Recombines all tokens back into a single token using the specified delimiter.
*/
public TokenCombiner(TokenStream in, int delimiter) {
super(in);
this.delimiter = delimiter;
}
int delimiter;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
private boolean firstToken = true;
int startOffset = 0;
#Override
public final boolean incrementToken() throws IOException {
while (true){
boolean eos = input.incrementToken(); //We have to process tokens even if they return end of file.
CharTermAttribute token = input.getAttribute(CharTermAttribute.class);
if (eos && token.length() == 0) break; //Break early to avoid extra whitespace.
if (firstToken){
startOffset = input.getAttribute(OffsetAttribute.class).startOffset();
firstToken = false;
}else{
termAtt.append(Character.toString((char)delimiter));
}
termAtt.append(token);
if (eos) break;
}
offsetAtt.setOffset(startOffset, input.getAttribute(OffsetAttribute.class).endOffset());
return false;
}
#Override
public void reset() throws IOException {
super.reset();
firstToken = true;
startOffset = 0;
}
}
I think the fundamental problem here, is that you must realize both TokenCombiner and the producer it consumes (input) share and reuse the same attributes! So token == termAtt always (try adding an assert!).
Man, that sucks if you have been coding for 36 hours on a weekend... try this:
public class TokenCombiner extends TokenFilter {
private final StringBuilder sb = new StringBuilder();
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
private final char separator;
private boolean consumed; // true if we already consumed
protected TokenCombiner(TokenStream input, char separator) {
super(input);
this.separator = separator;
}
#Override
public final boolean incrementToken() throws IOException {
if (consumed) {
return false; // don't call input.incrementToken() after it returns false
}
consumed = true;
int startOffset = 0;
int endOffset = 0;
boolean found = false; // true if we actually consumed any tokens
while (input.incrementToken()) {
if (!found) {
startOffset = offsetAtt.startOffset();
found = true;
}
sb.append(termAtt);
sb.append(separator);
endOffset = offsetAtt.endOffset();
}
if (found) {
assert sb.length() > 0; // always: because we append separator
sb.setLength(sb.length() - 1);
clearAttributes();
termAtt.setEmpty().append(sb);
offsetAtt.setOffset(startOffset, endOffset);
return true;
} else {
return false;
}
}
#Override
public void reset() throws IOException {
super.reset();
sb.setLength(0);
consumed = false;
}
}