Apache atlas column lineage not found - hive

Question

I use Hive2.1.1 and Atlas2.0.0.
I can found table lineage but not found column level lineage
the detailed information is shown below
For debug,
I try Restful first, nothing about column lineage.
[root#cent1 bin]# curl -X GET -u admin:admin http://cent1:21000/api/atlas/v2/lineage/27c81b16-b422-4479-84b9-2d643b5dba48
{"baseEntityGuid":"27c81b16-b422-4479-84b9-2d643b5dba48","lineageDirection":"BOTH","lineageDepth":3,"guidEntityMap":{},"relations":[]}
I print Kafka msg from ATLAS_HOOK topic below:
{
"version":{
"version":"1.0.0",
"versionParts":[
1
]
},
"msgCompressionKind":"NONE",
"msgSplitIdx":1,
"msgSplitCount":1,
"msgSourceIP":"192.168.10.128",
"msgCreatedBy":"root",
"msgCreationTime":1572687183793,
"message":{
"type":"ENTITY_CREATE_V2",
"user":"root",
"entities":{
"referredEntities":{
"-53640556650491":{
"typeName":"hive_column",
"attributes":{
"owner":"root",
"qualifiedName":"atlas1.pokes.bar#primary",
"name":"bar",
"comment":null,
"position":1,
"type":"string",
"table":{
"guid":"-53640556650488",
"typeName":"hive_table",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes#primary"
}
}
},
"guid":"-53640556650491",
"provenanceType":0,
"version":0,
"proxy":false
},
"-53640556650492":{
"typeName":"hive_table",
"attributes":{
"owner":"root",
"temporary":false,
"lastAccessTime":1572687176000,
"qualifiedName":"atlas1.pokes_create3#primary",
"columns":[
{
"guid":"-53640556650494",
"typeName":"hive_column",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes_create3.foo#primary"
}
},
{
"guid":"-53640556650495",
"typeName":"hive_column",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes_create3.bar#primary"
}
}
],
"tableType":"MANAGED_TABLE",
"sd":{
"guid":"-53640556650493",
"typeName":"hive_storagedesc",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes_create3#primary_storage"
}
},
"createTime":1572687176000,
"name":"pokes_create3",
"comment":null,
"partitionKeys":[
],
"parameters":{
"totalSize":"5812",
"numRows":"500",
"rawDataSize":"5312",
"COLUMN_STATS_ACCURATE":"{"BASIC_STATS":"true"}",
"numFiles":"1",
"transient_lastDdlTime":"1572687178"
},
"db":{
"guid":"-53640556650487",
"typeName":"hive_db",
"uniqueAttributes":{
"qualifiedName":"atlas1#primary"
}
},
"retention":0
},
"guid":"-53640556650492",
"provenanceType":0,
"version":0,
"proxy":false
},
"-53640556650490":{
"typeName":"hive_column",
"attributes":{
"owner":"root",
"qualifiedName":"atlas1.pokes.foo#primary",
"name":"foo",
"comment":null,
"position":0,
"type":"int",
"table":{
"guid":"-53640556650488",
"typeName":"hive_table",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes#primary"
}
}
},
"guid":"-53640556650490",
"provenanceType":0,
"version":0,
"proxy":false
},
"-53640556650495":{
"typeName":"hive_column",
"attributes":{
"owner":"root",
"qualifiedName":"atlas1.pokes_create3.bar#primary",
"name":"bar",
"comment":null,
"position":1,
"type":"string",
"table":{
"guid":"-53640556650492",
"typeName":"hive_table",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes_create3#primary"
}
}
},
"guid":"-53640556650495",
"provenanceType":0,
"version":0,
"proxy":false
},
"-53640556650493":{
"typeName":"hive_storagedesc",
"attributes":{
"qualifiedName":"atlas1.pokes_create3#primary_storage",
"storedAsSubDirectories":false,
"location":"hdfs://cent1:9000/user/hive/warehouse/atlas1.db/pokes_create3",
"compressed":false,
"inputFormat":"org.apache.hadoop.mapred.TextInputFormat",
"parameters":{
},
"outputFormat":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
"table":{
"guid":"-53640556650492",
"typeName":"hive_table",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes_create3#primary"
}
},
"serdeInfo":{
"typeName":"hive_serde",
"attributes":{
"serializationLib":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
"name":null,
"parameters":{
"serialization.format":"1"
}
}
},
"numBuckets":-1
},
"guid":"-53640556650493",
"provenanceType":0,
"version":0,
"proxy":false
},
"-53640556650494":{
"typeName":"hive_column",
"attributes":{
"owner":"root",
"qualifiedName":"atlas1.pokes_create3.foo#primary",
"name":"foo",
"comment":null,
"position":0,
"type":"int",
"table":{
"guid":"-53640556650492",
"typeName":"hive_table",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes_create3#primary"
}
}
},
"guid":"-53640556650494",
"provenanceType":0,
"version":0,
"proxy":false
},
"-53640556650488":{
"typeName":"hive_table",
"attributes":{
"owner":"root",
"temporary":false,
"lastAccessTime":1572663197000,
"qualifiedName":"atlas1.pokes#primary",
"columns":[
{
"guid":"-53640556650490",
"typeName":"hive_column",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes.foo#primary"
}
},
{
"guid":"-53640556650491",
"typeName":"hive_column",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes.bar#primary"
}
}
],
"tableType":"MANAGED_TABLE",
"sd":{
"guid":"-53640556650489",
"typeName":"hive_storagedesc",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes#primary_storage"
}
},
"createTime":1572663197000,
"name":"pokes",
"comment":null,
"partitionKeys":[
],
"parameters":{
"transient_lastDdlTime":"1572663225",
"totalSize":"5812",
"numRows":"0",
"rawDataSize":"0",
"numFiles":"1"
},
"db":{
"guid":"-53640556650487",
"typeName":"hive_db",
"uniqueAttributes":{
"qualifiedName":"atlas1#primary"
}
},
"retention":0
},
"guid":"-53640556650488",
"provenanceType":0,
"version":0,
"proxy":false
},
"-53640556650489":{
"typeName":"hive_storagedesc",
"attributes":{
"qualifiedName":"atlas1.pokes#primary_storage",
"storedAsSubDirectories":false,
"location":"hdfs://cent1:9000/user/hive/warehouse/atlas1.db/pokes",
"compressed":false,
"inputFormat":"org.apache.hadoop.mapred.TextInputFormat",
"parameters":{
},
"outputFormat":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
"table":{
"guid":"-53640556650488",
"typeName":"hive_table",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes#primary"
}
},
"serdeInfo":{
"typeName":"hive_serde",
"attributes":{
"serializationLib":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
"name":null,
"parameters":{
"serialization.format":"1"
}
}
},
"numBuckets":-1
},
"guid":"-53640556650489",
"provenanceType":0,
"version":0,
"proxy":false
},
"-53640556650487":{
"typeName":"hive_db",
"attributes":{
"owner":"root",
"ownerType":"USER",
"qualifiedName":"atlas1#primary",
"clusterName":"primary",
"name":"atlas1",
"description":null,
"location":"hdfs://cent1:9000/user/hive/warehouse/atlas1.db",
"parameters":{
}
},
"guid":"-53640556650487",
"provenanceType":0,
"version":0,
"proxy":false
}
},
"entities":[
{
"typeName":"hive_process",
"attributes":{
"outputs":[
{
"guid":"-53640556650492",
"typeName":"hive_table",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes_create3#primary"
}
}
],
"recentQueries":[
"create table pokes_create3 as select foo,bar from pokes"
],
"qualifiedName":"atlas1.pokes_create3#primary:1572687176000",
"inputs":[
{
"guid":"-53640556650488",
"typeName":"hive_table",
"uniqueAttributes":{
"qualifiedName":"atlas1.pokes#primary"
}
}
],
"name":"create table pokes_create3 as select foo,bar from pokes",
"queryText":"create table pokes_create3 as select foo,bar from pokes",
"operationType":"CREATETABLE_AS_SELECT",
"startTime":1572686979104,
"queryPlan":"Not Supported",
"endTime":1572687183754,
"userName":"root",
"queryId":"root_20191102172939_5de2901c-49e1-473c-ad2f-7e5f5b3251ee"
},
"guid":"-53640556650496",
"provenanceType":0,
"version":0,
"proxy":false
}
]
}
}
}
Where is the column lineage? How can I debug this further?

Answer 1

It seems that you need to have both of these Hive patches.
https://issues.apache.org/jira/browse/HIVE-13112 released in Hive 2.1.0
https://issues.apache.org/jira/browse/HIVE-14706 released in hive 2.2.0
As far as I understand, the first patch exposes Lineage information in case of CTAS and the second patch allows atlas plugin to pick it up from the hive hook.
According to the Atlas doc : https://atlas.apache.org/#/HookHive