lead/lag function in apache PIG - sql

Question

Is there a function in apache PIG that's similar to Lead/Lag function in SQL? Or any pig function that can look back to previous row of record?

Answer 1

Yes, there is pre-defined functionality. See the Over() and Stitch() methods in Piggybank. Over() has examples listed in the documentation.

Answer 2

Here is an alternative:
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.pig.EvalFunc;
import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
public class GenericLag2 extends EvalFunc<Tuple>{
private List<String> lagObjects = null;
#Override
public Tuple exec(Tuple input) throws IOException {
if (lagObjects == null) {
lagObjects = new ArrayList<String>();
return null;
}
try {
Tuple output = TupleFactory.getInstance().newTuple(lagObjects.size());
for (int i = 0; i < lagObjects.size(); i++) {
output.set(i, lagObjects.get(i));
}
lagObjects.clear();
for (int i = 0; i < input.size(); i++) {
lagObjects.add(input.get(i).toString());
}
return output;
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
#Override
public Schema outputSchema(Schema input) {
Schema tupleSchema = new Schema();
try {
for (int i = 0; i < input.size(); i++) {
tupleSchema.add(new FieldSchema("lag_" + i, DataType.CHARARRAY));
}
return new Schema(new FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input), tupleSchema, DataType.TUPLE));
} catch (FrontendException e) {
e.printStackTrace();
return null;
}
}
}
I assume this would be faster, but I'm not sure, as you would have to do the following:
...
C = ORDER A BY important_order_by_field, second_important_order_by_field
D = FOREACH B GENERATE
important_order_by_field
,second_important_order_by_field
,...
,FLATTEN(LAG(
string_field_to_lag
,int_field_to_lag
,date_field_to_lag
))
;
E = FOREACH D GENERATE
important_order_by_field
,second_important_order_by_field
,...
,string_field_to_lag
,(int) int_field_to_lag
,(date_field_to_lag IS NULL ?
null :
ToDate(SUBSTRING(REPLACE(date_field_to_lag, 'T', ' '), 0, 19), 'yyyy-MM-dd HH:mm:ss'))
as date_field_to_lag
;
DUMP E;

Answer 3

Ok here is my first shot at this. Mind you, I just started learning how to code UDFs today.
Maven's pom.xml file contains:
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.0.0-cdh4.1.0</version>
</dependency>
...
Java UDF Class:
import java.io.IOException;
import org.apache.pig.EvalFunc;
import org.apache.pig.data.Tuple;
public class GenericLag extends EvalFunc<String>{
private String lagObject = null;
#Override
public String exec(Tuple input) throws IOException {
try {
String returnObject = getLagObject();
setLagObject(input.get(0).toString());
return returnObject;
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
public String getLagObject() {
return lagObject;
}
public void setLagObject(String lagObject) {
this.lagObject = lagObject;
}
}
Initially, I had used Object instead of String everywhere that you see "String" above, but I received this error:
ERROR org.apache.pig.tools.grunt.Grunt - ERROR 2080: Foreach currently does not handle type Unknown
I had to issue setLagObject(input.get(0).toString()); instead of setLagObject(input.get(0); or I would have received errors like:
java.lang.ClassCastException: java.lang.Integer cannot be cast to java.lang.String
java.lang.ClassCastException: java.lang.Double cannot be cast to java.lang.String
java.lang.ClassCastException: org.joda.time.DateTime cannot be cast to java.lang.String
Here is how I use it in Pig:
REGISTER /path/to/compiled/file.jar
DEFINE LAG fully.qualified.domain.name.GenericLag();
A = LOAD '/hdfs/path/to/directory' USING PigStorage(',') AS (
important_order_by_field:int
,second_important_order_by_field:string
,...
,string_field_to_lag:string
,int_field_to_lag:int
,date_field_to_lag:string
);
B = FOREACH A GENERATE
important_order_by_field
,second_important_order_by_field
,...
,string_field_to_lag
,int_field_to_lag
,ToDate(date_field_to_lag, 'yyyy-MM-dd HH:mm:ss')
;
C = ORDER A BY important_order_by_field, second_important_order_by_field
D = FOREACH B GENERATE
important_order_by_field
,second_important_order_by_field
,...
,LAG(string_field_to_lag) AS lag_string
,(int) LAG(int_field_to_lag) AS lag_int
,(date_field_to_lag IS NULL ?
null :
ToDate(SUBSTRING(REPLACE(LAG(date_field_to_lag), 'T', ' ') ,0,19), 'yyyy-MM-dd HH:mm:ss')) AS lag_date
;
DUMP D;
If I did the last line like this:
ToDate(SUBSTRING(REPLACE(LAG(date_field_to_lag), 'T', ' ') ,0,19), 'yyyy-MM-dd HH:mm:ss') AS lag_date
It would return the following error
ERROR org.apache.pig.tools.grunt.Grunt - ERROR 1066: Unable to open iterator for alias LAGGED_RHODES. Backend error : null
Which when checking the logs reveals:
java.lang.NullPointerException
at org.joda.time.format.DateTimeFormatterBuilder$NumberFormatter.parseInto(DateTimeFormatterBuilder.java:1200)
because the first row will contain a null value.