Query Expansion lucene - lucene

I am new to lucene and I am trying to do query expansion.
I have referred to these two posts (first , second) and I've managed to reuse the code in a way that suits version 6.0.0, as the one in the previous is deprecated.
The issue is, either I'm not getting a results or I didn't access the results (expanded queries) appropriately.
Here is my code:
import com.sun.corba.se.impl.util.Version;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.text.ParseException;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.standard.ClassicTokenizer;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.synonym.SynonymFilter;
import org.apache.lucene.analysis.synonym.SynonymMap;
import org.apache.lucene.analysis.synonym.WordnetSynonymParser;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.util.*;
public class Graph extends Analyzer
{
protected static TokenStreamComponents createComponents(String fieldName, Reader reader) throws ParseException{
System.out.println("1");
// TODO Auto-generated method stub
Tokenizer source = new ClassicTokenizer();
source.setReader(reader);
TokenStream filter = new StandardFilter( source);
filter = new LowerCaseFilter(filter);
SynonymMap mySynonymMap = null;
try {
mySynonymMap = buildSynonym();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
filter = new SynonymFilter(filter, mySynonymMap, false);
return new TokenStreamComponents(source, filter);
}
private static SynonymMap buildSynonym() throws IOException, ParseException
{ System.out.print("build");
File file = new File("wn\\wn_s.pl");
InputStream stream = new FileInputStream(file);
Reader rulesReader = new InputStreamReader(stream);
SynonymMap.Builder parser = null;
parser = new WordnetSynonymParser(true, true, new StandardAnalyzer(CharArraySet.EMPTY_SET));
System.out.print(parser.toString());
((WordnetSynonymParser) parser).parse(rulesReader);
SynonymMap synonymMap = parser.build();
return synonymMap;
}
public static void main (String[] args) throws UnsupportedEncodingException, IOException, ParseException
{
Reader reader = new FileReader("C:\\input.txt"); // here I have the queries that I want to expand
TokenStreamComponents TSC = createComponents( "" , new StringReader("some text goes here"));
**System.out.print(TSC); //How to get the result from TSC????**
}
#Override
protected TokenStreamComponents createComponents(String string)
{
throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
}
}
Please suggest ways to help me access the expanded queries!

So, are you just trying to figure out how to iterate through the terms from the TokenStreamComponents in your main method?
Something like this:
TokenStreamComponents TSC = createComponents( "" , new StringReader("some text goes here"));
TokenStream stream = TSC.getTokenStream();
CharTermAttribute termattr = stream.addAttribute(CharTermAttribute.class);
stream.reset();
while (stream.incrementToken()) {
System.out.println(termattr.toString());
}

Related

Apache FOP the method newInstance(FopFactoryConfig) in the type FopFactory is not applicable for the arguments ()

I am getting the following error:
Exception in thread "main" java.lang.Error: Unresolved compilation problem: The method newInstance(FopFactoryConfig) in the type FopFactory is not applicable for the arguments ()
at fopdemo.fopvass.PDFHandler.createPDFFile(PDFHandler.java:42)
at fopdemo.fopvass.TestPDF.main(TestPDF.java:40)
This is my code:
package fopdemo.fopvass;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URL;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Marshaller;
import javax.xml.transform.Result;
import javax.xml.transform.Source;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerConfigurationException;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.TransformerFactoryConfigurationError;
import javax.xml.transform.dom.DOMResult;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.sax.SAXResult;
import javax.xml.transform.stream.StreamSource;
import org.apache.fop.apps.FOPException;
import org.apache.fop.apps.FOUserAgent;
import org.apache.fop.apps.Fop;
import org.apache.fop.apps.FopFactory;
import org.apache.fop.apps.MimeConstants;
public class PDFHandler {
public static final String EXTENSION = ".pdf";
public String PRESCRIPTION_URL = "template.xsl";
public String createPDFFile(ByteArrayOutputStream xmlSource, String templateFilePath) throws IOException {
File file = File.createTempFile("" + System.currentTimeMillis(), EXTENSION);
URL url = new File(templateFilePath + PRESCRIPTION_URL).toURI().toURL();
// creation of transform source
StreamSource transformSource = new StreamSource(url.openStream());
// create an instance of fop factory
FopFactory fopFactory = FopFactory.newInstance();
// a user agent is needed for transformation
FOUserAgent foUserAgent = fopFactory.newFOUserAgent();
// to store output
ByteArrayOutputStream pdfoutStream = new ByteArrayOutputStream();
StreamSource source = new StreamSource(new ByteArrayInputStream(xmlSource.toByteArray()));
Transformer xslfoTransformer;
try {
TransformerFactory transfact = TransformerFactory.newInstance();
xslfoTransformer = transfact.newTransformer(transformSource);
// Construct fop with desired output format
Fop fop;
try {
fop = fopFactory.newFop(MimeConstants.MIME_PDF, foUserAgent, pdfoutStream);
// Resulting SAX events (the generated FO)
// must be piped through to FOP
Result res = new SAXResult(fop.getDefaultHandler());
// Start XSLT transformation and FOP processing
try {
// everything will happen here..
xslfoTransformer.transform(source, res);
// if you want to save PDF file use the following code
OutputStream out = new java.io.FileOutputStream(file);
out = new java.io.BufferedOutputStream(out);
FileOutputStream str = new FileOutputStream(file);
str.write(pdfoutStream.toByteArray());
str.close();
out.close();
} catch (TransformerException e) {
e.printStackTrace();
}
} catch (FOPException e) {
e.printStackTrace();
}
} catch (TransformerConfigurationException e) {
e.printStackTrace();
} catch (TransformerFactoryConfigurationError e) {
e.printStackTrace();
}
return file.getPath();
}
public ByteArrayOutputStream getXMLSource(EmployeeData data) throws Exception {
JAXBContext context;
ByteArrayOutputStream outStream = new ByteArrayOutputStream();
try {
context = JAXBContext.newInstance(EmployeeData.class);
Marshaller m = context.createMarshaller();
m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE);
m.marshal(data, System.out);
m.marshal(data, outStream);
} catch (JAXBException e) {
e.printStackTrace();
}
return outStream;
}
}
This is where it is called:
package fopdemo.fopvass;
import java.io.ByteArrayOutputStream;
import java.util.ArrayList;
/**
* #author Debasmita.Sahoo
*
*/
public class TestPDF {
public static void main(String args[]){
System.out.println("Hi Testing");
ArrayList employeeList = new ArrayList();
String templateFilePath ="C:/Paula/Proyectos/fop/fopvass/resources/";
Employee e1= new Employee();
e1.setName("Debasmita1 Sahoo");
e1.setEmployeeId("10001");
e1.setAddress("Pune");
employeeList.add(e1);
Employee e2= new Employee();
e2.setName("Debasmita2 Sahoo");
e2.setEmployeeId("10002");
e2.setAddress("Test");
employeeList.add(e2);
Employee e3= new Employee();
e3.setName("Debasmita3 Sahoo");
e3.setEmployeeId("10003");
e3.setAddress("Mumbai");
employeeList.add(e3);
EmployeeData data = new EmployeeData();
data.setEemployeeList(employeeList);
PDFHandler handler = new PDFHandler();
try {
ByteArrayOutputStream streamSource = handler.getXMLSource(data);
handler.createPDFFile(streamSource,templateFilePath);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
The following line won't compile:
FopFactory fopFactory = FopFactory.newInstance();
The method newInstance() requires a parameter. From the documentation (provided by Mathias Muller), under 'Basic Usage', that parameter refers to a configuration file:
FopFactory fopFactory = FopFactory.newInstance(
new File( "C:/Temp/fop.xconf" ) );
You need to provide it a File object. Alternatively, it is also possible to create a FopFactory instance by providing a URI to resolve relative URIs in the input file (because SVG files reference other SVG files). As the documentation suggests:
FopFactory fopFactory = FopFactory.newInstance(
new File(".").toURI() );
The user agent is the entity that allows you to interact with a single rendering run, i.e. the processing of a single document. If you wish to customize the user agent's behaviour, the first step is to create your own instance of FOUserAgent using the appropriate factory method on FopFactory and pass that to the factory method that will create a new Fop instance.
FopFactory.newInstance() support
FopFactory.newInstance(File)
FopFactory.newInstance(Uri)
FopFactory.newInstance(Uri, InputStream)
FopFactory.newInstance(FopFactoryConfig)
I successfully implemented on Windows and Mac OS (i.e. a Unix\Linux like NFS) the following code
URI uri = new File(config.getRoot()).toPath().toUri();
FopFactory fopFactory = FopFactory.newInstance(uri);
On the other hand, this does not work (and I don't understand why)
URI uri = new File(config.getRoot()).toUri();

Lucene - apply TokenFilter to a String and get result

I want to debug Lucene token filters and see results. How is it possible to apply a token filter to a token stream in order to see result?
(using Lucene 4.10.3)
import java.io.IOException;
import java.io.StringReader;
import java.util.Iterator;
import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
public class TokenFilterExample {
public static void main(String[] args) throws IOException {
// 1] Create token stream
StringReader r = new StringReader("Hello World");
StandardTokenizer s = new StandardTokenizer(r);
// Create lower-case token filter
LowerCaseFilter f = new LowerCaseFilter(s);
// Print result
System.out.println(??????);
// close
f.close();
s.close();
}
}
The solution is this:
// Print result
f.reset();
Iterator it = f.getAttributeClassesIterator();
while (f.incrementToken()) {
System.out.println(f
.getAttribute(CharTermAttribute.class));
}

Parsing content from Word document using docx4j

Thanks to a previous answer, I'm now able to read my password-protected Word 2010 documents. (I have to translate them one by one from .doc to .docx. They go back to 1994, but that's okay.)
I wrote a simple Java class to get started:
package model.docx4j;
import model.JournalEntry;
import model.JournalEntryFactory;
import org.docx4j.openpackaging.exceptions.Docx4JException;
import org.docx4j.openpackaging.packages.OpcPackage;
import org.docx4j.openpackaging.parts.Parts;
import java.io.IOException;
import java.io.InputStream;
import java.security.GeneralSecurityException;
import java.util.LinkedList;
import java.util.List;
/**
* JournalEntryFactoryImpl using docx4j
* #author Michael
* #link
* #since 9/8/12 12:44 PM
*/
public class JournalEntryFactoryImpl implements JournalEntryFactory {
#Override
public List<JournalEntry> getEntries(InputStream inputStream, String password) throws IOException, GeneralSecurityException {
List<JournalEntry> journalEntries = new LinkedList<JournalEntry>();
if (inputStream != null) {
try {
OpcPackage opcPackage = OpcPackage.load(inputStream, password);
Parts parts = opcPackage.getParts();
} catch (Docx4JException e) {
LOGGER.error("Could not load document into docx4j", e);
throw new IOException(e);
}
}
return journalEntries;
}
}
And a JUnit test to drive it:
package model.docx4j;
import model.JournalEntry;
import model.JournalEntryFactory;
import org.docx4j.openpackaging.exceptions.Docx4JException;
import org.docx4j.openpackaging.packages.OpcPackage;
import org.docx4j.openpackaging.parts.Parts;
import java.io.IOException;
import java.io.InputStream;
import java.security.GeneralSecurityException;
import java.util.LinkedList;
import java.util.List;
/**
* JournalEntryFactoryImpl using docx4j
* #author Michael
* #link
* #since 9/8/12 12:44 PM
*/
public class JournalEntryFactoryImpl implements JournalEntryFactory {
#Override
public List<JournalEntry> getEntries(InputStream inputStream, String password) throws IOException, GeneralSecurityException {
List<JournalEntry> journalEntries = new LinkedList<JournalEntry>();
if (inputStream != null) {
try {
OpcPackage opcPackage = OpcPackage.load(inputStream, password);
Parts parts = opcPackage.getParts();
} catch (Docx4JException e) {
LOGGER.error("Could not load document into docx4j", e);
throw new IOException(e);
}
}
return journalEntries;
}
}
I put a breakpoint into the test to see what docx4j was doing once it read my document. I see a list of 8 parts, but I walked through the tree without finding the content.
Each document consists of a page with a date and content, but I can't find pages. Where do they live?
The main document content lives in the "main document part", which is often named "/word/document.xml".
The usual way to get it with docx4j is:
WordprocessingMLPackage wordMLPackage = (WordprocessingMLPackage)opcPackage;
MainDocumentPart documentPart = wordMLPackage.getMainDocumentPart();
but you'd expect your approach to work as well.

Selenium tests for custom portlets

Can anyone provide me a link/document with information on how to write and test custom liferay portlets with Selenium.
I am using Liferay 6.1EE
Thanks
Same as with other tests, assuming you want to run as JUnit test
package org.ood.selenium.test;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.IncorrectnessListener;
import com.gargoylesoftware.htmlunit.ScriptException;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlButton;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.javascript.JavaScriptErrorListener;
import com.thoughtworks.selenium.SeleneseTestBase;
import com.thoughtworks.selenium.Selenium;
import org.openqa.selenium.firefox.FirefoxDriver;
import org.openqa.selenium.htmlunit.HtmlUnitDriver;
import org.openqa.selenium.remote.DesiredCapabilities;
import org.openqa.selenium.By;
import org.openqa.selenium.OutputType;
import org.openqa.selenium.TakesScreenshot;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebDriverBackedSelenium;
import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.LogFactory;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.File;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.logging.Level;
import java.util.regex.Pattern;
public class MyTest extends SeleneseTestBase {
WebDriver driver =null;
#Before
public void setUp() throws Exception {
DesiredCapabilities dc = DesiredCapabilities.firefox();
// dc.setCapability(FirefoxDriver.BINARY, new
//File("C:/Program Files (x86)/Mozilla Firefox/firefox.exe").getAbsolutePath());
dc.setJavascriptEnabled(true);
// driver= new HtmlUnitDriver(true);//Not properly working
driver = new FirefoxDriver(dc);
String baseUrl = "http://localhost:8080/web/guest/home";
selenium = new WebDriverBackedSelenium(driver, baseUrl);
}
/* #Test
public void homePage() throws Exception {
final WebClient webClient = new WebClient();
//HTMLUNit throws lots of errors
webClient.setJavaScriptEnabled(true);
webClient.setJavaScriptErrorListener(new JavaScriptErrorListener() {
public void timeoutError(HtmlPage htmlPage, long allowedTime,
long executionTime) {
// TODO Auto-generated method stub
}
public void scriptException(HtmlPage htmlPage,
ScriptException scriptException) {
System.out.println(scriptException.getMessage());
}
public void malformedScriptURL(HtmlPage htmlPage, String url,
MalformedURLException malformedURLException) {
System.out.println(url);
}
public void loadScriptError(HtmlPage htmlPage, URL scriptUrl,
Exception exception) {
// TODO Auto-generated method stub
}
});
//webClient.setThrowExceptionOnFailingStatusCode(false);
webClient.setThrowExceptionOnScriptError(false);
final HtmlPage page = webClient.getPage("http://localhost:8080/web/guest/home");
webClient.getCache().clear();
page.getElementById("sign-in").click();
page.getElementById("_58_login").type("test#liferay.com");
page.getElementById("_58_password").type("test#liferay.com");
final HtmlButton submit = (HtmlButton)page.getByXPath("//input").get(0);
submit.click();
//assertEquals("HtmlUnit - Welcome to HtmlUnit", page.getTitleText());
System.out.println("Title= " +page.getTitleText());
final String pageAsXml = page.asXml();
// assertTrue(pageAsXml.contains("<body class=\"composite\">"));
System.out.println("pageAsXml=" +pageAsXml);
final String pageAsText = page.asText();
//assertTrue(pageAsText.contains("Support for the HTTP and HTTPS protocols"));
System.out.println("pageAsText="+pageAsText);
webClient.closeAllWindows();
}*/
#Test //Using Graphical GUI
public void testMytest() throws Exception {
//driver.get("http://localhost:8080/web/guest/home");
selenium.open("/");
//selenium.click("id=sign-in");
driver.findElement(By.id("sign-in")).click();
driver.findElement(By.name("_58_login")).clear();
driver.findElement(By.name("_58_login")).sendKeys("test#liferay.com");
driver.findElement(By.name("_58_password")).sendKeys("123");
//driver.findElement(By.name("_58_login")).sendKeys("test#nsn.com");
//driver.findElement(By.name("_58_password")).sendKeys("test");
driver.findElement(By.className("aui-button-input-submit")).submit();
//selenium.setSpeed("1000");
//selenium.waitForPageToLoad("5000");
//Select a nested div( the NodeTree)
driver.findElement(By.xpath("//div[contains(#class,'aui-tree-node-content')" +
" and (contains(.,'TreeNodeX'))]//div[contains(#class,'aui-tree-hitarea')]"))
.click();
//selenium.captureScreenshot("d:/_del/sel_screen.png");//this throws an error
//workaround
File scrFile = ((TakesScreenshot)driver).getScreenshotAs(OutputType.FILE);
FileUtils.copyFile(scrFile, new File("d:/_del/sel_screen.png"));
System.out.println("Bodytex=[" +selenium.getBodyText() +"]End Body Text");
String[] titles= selenium.getAllWindowTitles();
System.out.println("Titles Start-");
for (int i = 0; i < titles.length; i++) {
System.out.println(titles[i]);
}
System.out.println("Titles End");
verifyEquals(true, selenium.getBodyText().contains("Site View"));
}
#After
public void tearDown() throws Exception {
selenium.stop();
}
}

Lucene Highlighter with stemming analyzer

I am using Lucene's Highlighter class to highlight fragments of matched search results and it works well. I would like to switch from searching with the StandardAnalyzer to the EnglishAnalyzer, which will perform stemming of terms.
The search results are good, but now the highlighter doesn't always find a match. Here's an example of what I'm looking at:
document field text 1: Everyone likes goats.
document field text 2: I have a goat that eats everything.
Using the EnglishAnalyzer and searching for "goat", both documents are matched, but the highlighter is only able to find a matched fragment from document 2. Is there a way to have the highlighter return data for both documents?
I understand that the characters are different for the tokens, but the same tokens are still there, so it seems reasonable for it to just highlight whatever token is present at that location.
If it helps, this is using Lucene 3.5.
I found a solution to this problem. I changed from using the Highlighter class to using the FastVectorHighlighter. It looks like I'll pick up some speed improvements too (at the expense of storage of term vector data). For the benefit of anyone coming across this question later, here's a unit test showing how this all works together:
package com.sample.index;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.en.EnglishAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.vectorhighlight.*;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.Version;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import static junit.framework.Assert.assertEquals;
public class TestIndexStuff {
public static final String FIELD_NORMAL = "normal";
public static final String[] PRE_TAGS = new String[]{"["};
public static final String[] POST_TAGS = new String[]{"]"};
private IndexSearcher searcher;
private Analyzer analyzer = new EnglishAnalyzer(Version.LUCENE_35);
#Before
public void init() throws IOException {
RAMDirectory idx = new RAMDirectory();
IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_35, analyzer);
IndexWriter writer = new IndexWriter(idx, config);
addDocs(writer);
writer.close();
searcher = new IndexSearcher(IndexReader.open(idx));
}
private void addDocs(IndexWriter writer) throws IOException {
for (String text : new String[] {
"Pretty much everyone likes goats.",
"I have a goat that eats everything.",
"goats goats goats goats goats"}) {
Document doc = new Document();
doc.add(new Field(FIELD_NORMAL, text, Field.Store.YES,
Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
writer.addDocument(doc);
}
}
private FastVectorHighlighter makeHighlighter() {
FragListBuilder fragListBuilder = new SimpleFragListBuilder(200);
FragmentsBuilder fragmentBuilder = new SimpleFragmentsBuilder(PRE_TAGS, POST_TAGS);
return new FastVectorHighlighter(true, true, fragListBuilder, fragmentBuilder);
}
#Test
public void highlight() throws ParseException, IOException {
Query query = new QueryParser(Version.LUCENE_35, FIELD_NORMAL, analyzer)
.parse("goat");
FastVectorHighlighter highlighter = makeHighlighter();
FieldQuery fieldQuery = highlighter.getFieldQuery(query);
TopDocs topDocs = searcher.search(query, 10);
List<String> fragments = new ArrayList<String>();
for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
fragments.add(highlighter.getBestFragment(fieldQuery, searcher.getIndexReader(),
scoreDoc.doc, FIELD_NORMAL, 10000));
}
assertEquals(3, fragments.size());
assertEquals("[goats] [goats] [goats] [goats] [goats]", fragments.get(0).trim());
assertEquals("Pretty much everyone likes [goats].", fragments.get(1).trim());
assertEquals("I have a [goat] that eats everything.", fragments.get(2).trim());
}
}