I'm trying to convert a SQL query to a relational algebra expression using the Apache Calcite SqlToRelConverter.
It works fine for this query (quotes are for ensuring lowercase):
queryToRelationalAlgebraRoot("SELECT \"country\" FROM \"mytable\"")
But on this query it fails:
queryToRelationalAlgebraRoot("SELECT \"country\", SUM(\"salary\") FROM \"mytable\" GROUP BY \"country\"")
with this error:
org.apache.calcite.sql.validate.SqlValidatorException: No match found for function signature SUM(<NUMERIC>)
It seems that somehow the SQL validator doesn't have aggregation functions like sum or count registered.
case class Income(id: Int, salary: Double, country: String)
class SparkDataFrameTable(df: DataFrame) extends AbstractTable {
def getRowType(typeFactory: RelDataTypeFactory): RelDataType = {
val typeList = df.schema.fields.map {
field => field.dataType match {
case t: StringType => typeFactory.createSqlType(SqlTypeName.VARCHAR)
case t: IntegerType => typeFactory.createSqlType(SqlTypeName.INTEGER)
case t: DoubleType => typeFactory.createSqlType(SqlTypeName.DOUBLE)
}
}.toList.asJava
val fieldNameList = df.schema.fieldNames.toList.asJava
typeFactory.createStructType(typeList, fieldNameList)
}
}
object RelationalAlgebra {
def queryToRelationalAlgebraRoot(query: String): RelRoot = {
val sqlParser = SqlParser.create(query)
val sqlParseTree = sqlParser.parseQuery()
val frameworkConfig = Frameworks.newConfigBuilder().build()
val planner = new PlannerImpl(frameworkConfig)
val rootSchema = CalciteSchema.createRootSchema(true, true)
// some sample data for testing
val inc1 = new Income(1, 100000, "USA")
val inc2 = new Income(2, 110000, "USA")
val inc3 = new Income(3, 80000, "Canada")
val spark = SparkSession.builder().master("local").getOrCreate()
import spark.implicits._
val df = Seq(inc1, inc2, inc3).toDF()
rootSchema.add("mytable", new SparkDataFrameTable(df))
val defaultSchema = List[String]().asJava
val calciteConnectionConfigProperties = new Properties()
val calciteConnectionConfigImpl = new CalciteConnectionConfigImpl(calciteConnectionConfigProperties)
val sqlTypeFactoryImpl = new SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT)
val calciteCatelogReader = new CalciteCatalogReader(rootSchema, defaultSchema, sqlTypeFactoryImpl, calciteConnectionConfigImpl)
val defaultValidator = SqlValidatorUtil.newValidator(new SqlStdOperatorTable(), calciteCatelogReader, sqlTypeFactoryImpl, SqlConformanceEnum.LENIENT)
val relExpressionOptimizationCluster = RelOptCluster.create(new VolcanoPlanner(), new RexBuilder(sqlTypeFactoryImpl))
val sqlToRelConfig = SqlToRelConverter.configBuilder().build()
val sqlToRelConverter = new SqlToRelConverter(planner, defaultValidator, calciteCatelogReader, relExpressionOptimizationCluster, StandardConvertletTable.INSTANCE, sqlToRelConfig)
sqlToRelConverter.convertQuery(sqlParseTree, true, true)
}
}
The problem with the code is that new SqlStdOperatorTable() creates a validator which is not initialized. The correct way to use SqlStdOperatorTable is to use SqlStdOperatorTable.instance().
I found the solution after emailing the dev#calcite.apache.org mailing list. I would like to thank Yuzhao Chen for looking into the question I had and pointing out the problem with my code.
I am not familiar with the api but your SQL needs group by country. And if a tool is to take this output and use it, it will probably require that you name the column too with an alias.
I have the following simplied definition of an addition operation over a field:
import inox._
import inox.trees.{forall => _, _}
import inox.trees.dsl._
object Field {
val element = FreshIdentifier("element")
val zero = FreshIdentifier("zero")
val one = FreshIdentifier("one")
val elementADT = mkSort(element)()(Seq(zero, one))
val zeroADT = mkConstructor(zero)()(Some(element)) {_ => Seq()}
val oneADT = mkConstructor(one)()(Some(element)) {_ => Seq()}
val addID = FreshIdentifier("add")
val addFunction = mkFunDef(addID)("element") { case Seq(eT) =>
val args: Seq[ValDef] = Seq("f1" :: eT, "f2" :: eT)
val retType: Type = eT
val body: Seq[Variable] => Expr = { case Seq(f1,f2) =>
//do the addition for this field
f1 //do something better...
}
(args, retType, body)
}
//-------Helper functions for arithmetic operations and zero element of field----------------
implicit class ExprOperands(private val lhs: Expr) extends AnyVal{
def +(rhs: Expr): Expr = E(addID)(T(element)())(lhs, rhs)
}
}
I'd like this operation to be used with infix notation and the current solution that I find to do so in Scala is given here. So that's why I'm including the implicit class at the bottom.
Say now I want to use this definition of addition:
import inox._
import inox.trees.{forall => _, _}
import inox.trees.dsl._
import welder._
object Curve{
val affinePoint = FreshIdentifier("affinePoint")
val infinitePoint = FreshIdentifier("infinitePoint")
val finitePoint = FreshIdentifier("finitePoint")
val first = FreshIdentifier("first")
val second = FreshIdentifier("second")
val affinePointADT = mkSort(affinePoint)("F")(Seq(infinitePoint,finitePoint))
val infiniteADT = mkConstructor(infinitePoint)("F")(Some(affinePoint))(_ => Seq())
val finiteADT = mkConstructor(finitePoint)("F")(Some(affinePoint)){ case Seq(fT) =>
Seq(ValDef(first, fT), ValDef(second, fT))
}
val F = T(Field.element)()
val affine = T(affinePoint)(F)
val infinite = T(infinitePoint)(F)
val finite = T(finitePoint)(F)
val onCurveID = FreshIdentifier("onCurve")
val onCurveFunction = mkFunDef(onCurveID)() { case Seq() =>
val args: Seq[ValDef] = Seq("p" :: affine, "a" :: F, "b" :: F)
val retType: Type = BooleanType
val body: Seq[Variable] => Expr = { case Seq(p,a,b) =>
if_(p.isInstOf(finite)){
val x: Expr = p.asInstOf(finite).getField(first)
val y: Expr = p.asInstOf(finite).getField(second)
x === y+y
} else_ {
BooleanLiteral(true)
}
}
(args, retType, body)
}
//---------------------------Registering elements-----------------------------------
val symbols = NoSymbols
.withADTs(Seq(affinePointADT,
infiniteADT,
finiteADT,
Field.zeroADT,
Field.oneADT,
Field.elementADT))
.withFunctions(Seq(Field.addFunction,
onCurveFunction))
val program = InoxProgram(Context.empty, symbols)
val theory = theoryOf(program)
import theory._
val expr = (E(BigInt(1)) + E(BigInt(1))) === E(BigInt(2))
val theorem: Theorem = prove(expr)
}
This won't compile giving the following error:
java.lang.ExceptionInInitializerError
at Main$.main(Main.scala:4)
at Main.main(Main.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
Caused by: inox.ast.TypeOps$TypeErrorException: Type error: if (p.isInstanceOf[finitePoint[element]]) {
p.asInstanceOf[finitePoint[element]].first == p.asInstanceOf[finitePoint[element]].second + p.asInstanceOf[finitePoint[element]].second
} else {
true
}, expected Boolean, found <untyped>
at inox.ast.TypeOps$TypeErrorException$.apply(TypeOps.scala:24)
at inox.ast.TypeOps$class.typeCheck(TypeOps.scala:264)
at inox.ast.SimpleSymbols$SimpleSymbols.typeCheck(SimpleSymbols.scala:12)
at inox.ast.Definitions$AbstractSymbols$$anonfun$ensureWellFormed$2.apply(Definitions.scala:166)
at inox.ast.Definitions$AbstractSymbols$$anonfun$ensureWellFormed$2.apply(Definitions.scala:165)
at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:733)
at scala.collection.immutable.Map$Map2.foreach(Map.scala:137)
at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732)
at inox.ast.Definitions$AbstractSymbols$class.ensureWellFormed(Definitions.scala:165)
at inox.ast.SimpleSymbols$SimpleSymbols.ensureWellFormed$lzycompute(SimpleSymbols.scala:12)
at inox.ast.SimpleSymbols$SimpleSymbols.ensureWellFormed(SimpleSymbols.scala:12)
at inox.solvers.unrolling.AbstractUnrollingSolver$class.assertCnstr(UnrollingSolver.scala:129)
at inox.solvers.SolverFactory$$anonfun$getFromName$1$$anon$1.inox$tip$TipDebugger$$super$assertCnstr(SolverFactory.scala:115)
at inox.tip.TipDebugger$class.assertCnstr(TipDebugger.scala:52)
at inox.solvers.SolverFactory$$anonfun$getFromName$1$$anon$1.assertCnstr(SolverFactory.scala:115)
at inox.solvers.SolverFactory$$anonfun$getFromName$1$$anon$1.assertCnstr(SolverFactory.scala:115)
at welder.Solvers$class.prove(Solvers.scala:51)
at welder.package$$anon$1.prove(package.scala:10)
at welder.Solvers$class.prove(Solvers.scala:23)
at welder.package$$anon$1.prove(package.scala:10)
at Curve$.<init>(curve.scala:61)
at Curve$.<clinit>(curve.scala)
at Main$.main(Main.scala:4)
at Main.main(Main.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
In fact, what is happening is that in the expression x === y+y the + is not being applied correctly so that it is untyped. I recall that inside the objects of Welder proofs one cannot define nested objects or classes I don't know if this has to do with it.
Anyways, do I have to forget about using infix notation in my code for Welder or there is a solution to this?
The issue here is that the implicit class you defined is not visible when you create y+y (you would need to import Field._ for it to be visible).
I don't remember exactly how implicit resolution takes place in Scala, so maybe adding import Field._ inside the Curve object will override the + that comes from the inox DSL (that's the one being applied when you write y+y, giving you an arithmetic plus expression that expects integer arguments, hence the type error). Otherwise, you'll unfortunately have ambiguity in the implicit resolution, and I'm not sure it's possible to use the infix + operator in that case without giving up the whole dsl.
I have defined a custom ToEntityMarshaller for type Organisation. When requesting localhost:8080/organisations it return an empty JSON array. Only when I remove the implicit def organisationMarshaller: ToEntityMarshaller[Organisation] it return the correct representation of the stream.
Anybody has an idea what is going on here?
import akka.NotUsed
import akka.actor.ActorSystem
import akka.http.scaladsl.Http
import akka.http.scaladsl.common.{EntityStreamingSupport, JsonEntityStreamingSupport}
import akka.http.scaladsl.model.{HttpEntity, StatusCodes, _}
import akka.http.scaladsl.server.Directives._
import akka.stream.ActorMaterializer
import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
import akka.http.scaladsl.marshalling.{Marshaller, ToEntityMarshaller, ToResponseMarshaller}
import akka.http.scaladsl.model.TransferEncodings.gzip
import akka.http.scaladsl.model.headers.{HttpEncoding, HttpEncodings}
import akka.stream.scaladsl.{Flow, Source}
import akka.util.ByteString
import spray.json.DefaultJsonProtocol
import spray.json.DefaultJsonProtocol._
import scala.concurrent.Future
import scala.io.StdIn
import scala.util.Random
final case class Organisation(name: String, id: String)
trait Protocols extends DefaultJsonProtocol {
import spray.json._
implicit val organisationFormat = jsonFormat2(Organisation)
val `vnd.example.api.v1+json` =
MediaType.applicationWithFixedCharset("vnd.example.api.v1+json", HttpCharsets.`UTF-8`)
// -- WORKS AFTER REMOVING THIS DECLARATION --
implicit def organisationMarshaller: ToEntityMarshaller[Organisation] = Marshaller.oneOf(
Marshaller.withFixedContentType(`vnd.example.api.v1+json`) { organisation =>
HttpEntity(`vnd.example.api.v1+json`, organisation.toJson.compactPrint)
})
}
object Server extends App with Protocols {
implicit val system = ActorSystem("api")
implicit val materializer = ActorMaterializer()
implicit val executionContext = system.dispatcher
implicit val jsonStreamingSupport: JsonEntityStreamingSupport = EntityStreamingSupport.json()
.withParallelMarshalling(parallelism = 10, unordered = false)
// (fake) async database query api
def dummyOrganisation(id: String) = Organisation(s"Organisation $id", id.toString)
def fetchOrganisation(id: String): Future[Option[Organisation]] = Future(Some(dummyOrganisation(id)))
def fetchOrganisations(): Source[Organisation, NotUsed] = Source.fromIterator(() => Iterator.fill(10000) {
val id = Random.nextInt()
dummyOrganisation(id.toString)
})
val route =
encodeResponse {
pathPrefix("organisations") {
get {
val organisations = fetchOrganisations()
complete(organisations)
}
}
}
val bindingFuture = Http().bindAndHandle(route, "localhost", 8080)
println(s"Server online at http://localhost:8080/\nPress RETURN to stop...")
StdIn.readLine()
bindingFuture.flatMap(_.unbind()).onComplete(_ => system.terminate())
}
In the spark docs it's clear how to create parquet files from RDD of your own case classes; (from the docs)
val people: RDD[Person] = ??? // An RDD of case class objects, from the previous example.
// The RDD is implicitly converted to a SchemaRDD by createSchemaRDD, allowing it to be stored using Parquet.
people.saveAsParquetFile("people.parquet")
But not clear how to convert back, really we want a method readParquetFile where we can do:
val people: RDD[Person] = sc.readParquestFile[Person](path)
where those values of the case class are defined are those which are read by the method.
An easy way is to provide your own converter (Row) => CaseClass. This is a bit more manual, but if you know what you are reading it should be quite straightforward.
Here is an example:
import org.apache.spark.sql.SchemaRDD
case class User(data: String, name: String, id: Long)
def sparkSqlToUser(r: Row): Option[User] = {
r match {
case Row(time: String, name: String, id: Long) => Some(User(time,name, id))
case _ => None
}
}
val parquetData: SchemaRDD = sqlContext.parquetFile("hdfs://localhost/user/data.parquet")
val caseClassRdd: org.apache.spark.rdd.RDD[User] = parquetData.flatMap(sparkSqlToUser)
The best solution I've come up with that requires the least amount of copy and pasting for new classes is as follows (I'd still like to see another solution though)
First you have to define your case class, and a (partially) reusable factory method
import org.apache.spark.sql.catalyst.expressions
case class MyClass(fooBar: Long, fred: Long)
// Here you want to auto gen these functions using macros or something
object Factories extends java.io.Serializable {
def longLong[T](fac: (Long, Long) => T)(row: expressions.Row): T =
fac(row(0).asInstanceOf[Long], row(1).asInstanceOf[Long])
}
Some boiler plate which will already be available
import scala.reflect.runtime.universe._
val sqlContext = new org.apache.spark.sql.SQLContext(sc)
import sqlContext.createSchemaRDD
The magic
import scala.reflect.ClassTag
import org.apache.spark.sql.SchemaRDD
def camelToUnderscores(name: String) =
"[A-Z]".r.replaceAllIn(name, "_" + _.group(0).toLowerCase())
def getCaseMethods[T: TypeTag]: List[String] = typeOf[T].members.sorted.collect {
case m: MethodSymbol if m.isCaseAccessor => m
}.toList.map(_.toString)
def caseClassToSQLCols[T: TypeTag]: List[String] =
getCaseMethods[T].map(_.split(" ")(1)).map(camelToUnderscores)
def schemaRDDToRDD[T: TypeTag: ClassTag](schemaRDD: SchemaRDD, fac: expressions.Row => T) = {
val tmpName = "tmpTableName" // Maybe should use a random string
schemaRDD.registerAsTable(tmpName)
sqlContext.sql("SELECT " + caseClassToSQLCols[T].mkString(", ") + " FROM " + tmpName)
.map(fac)
}
Example use
val parquetFile = sqlContext.parquetFile(path)
val normalRDD: RDD[MyClass] =
schemaRDDToRDD[MyClass](parquetFile, Factories.longLong[MyClass](MyClass.apply))
See also:
http://apache-spark-user-list.1001560.n3.nabble.com/Spark-SQL-Convert-SchemaRDD-back-to-RDD-td9071.html
Though I failed to find any example or documentation by following the JIRA link.
there is a simple method to convert schema rdd to rdd using pyspark in Spark 1.2.1.
sc = SparkContext() ## create SparkContext
srdd = sqlContext.sql(sql)
c = srdd.collect() ## convert rdd to list
rdd = sc.parallelize(c)
there must be similar approach using scala.
Very crufty attempt. Very unconvinced this will have decent performance. Surely there must a macro-based alternative...
import scala.reflect.runtime.universe.typeOf
import scala.reflect.runtime.universe.MethodSymbol
import scala.reflect.runtime.universe.NullaryMethodType
import scala.reflect.runtime.universe.TypeRef
import scala.reflect.runtime.universe.Type
import scala.reflect.runtime.universe.NoType
import scala.reflect.runtime.universe.termNames
import scala.reflect.runtime.universe.runtimeMirror
schemaRdd.map(row => RowToCaseClass.rowToCaseClass(row.toSeq, typeOf[X], 0))
object RowToCaseClass {
// http://dcsobral.blogspot.com/2012/08/json-serialization-with-reflection-in.html
def rowToCaseClass(record: Seq[_], t: Type, depth: Int): Any = {
val fields = t.decls.sorted.collect {
case m: MethodSymbol if m.isCaseAccessor => m
}
val values = fields.zipWithIndex.map {
case (field, i) =>
field.typeSignature match {
case NullaryMethodType(sig) if sig =:= typeOf[String] => record(i).asInstanceOf[String]
case NullaryMethodType(sig) if sig =:= typeOf[Int] => record(i).asInstanceOf[Int]
case NullaryMethodType(sig) =>
if (sig.baseType(typeOf[Seq[_]].typeSymbol) != NoType) {
sig match {
case TypeRef(_, _, args) =>
record(i).asInstanceOf[Seq[Seq[_]]].map {
r => rowToCaseClass(r, args(0), depth + 1)
}.toSeq
}
} else {
sig match {
case TypeRef(_, u, _) =>
rowToCaseClass(record(i).asInstanceOf[Seq[_]], sig, depth + 1)
}
}
}
}.asInstanceOf[Seq[Object]]
val mirror = runtimeMirror(t.getClass.getClassLoader)
val ctor = t.member(termNames.CONSTRUCTOR).asMethod
val klass = t.typeSymbol.asClass
val method = mirror.reflectClass(klass).reflectConstructor(ctor)
method.apply(values: _*)
}
}
New to Scala, but experienced in C++ I'm trying to implement (possibly misguidedly) a small library on top of the sqlite4java library to allow me to auto-fill tuples of abritrary type from query rows (rows whose column types are each compatible with the respective tuple element type).
In C++ I normally implement this using boost::tuples and compile-time template recursion (terminated using template specialisation as shown below). Boost tuples are implemented very similarly to Haskell HLists. The pattern would be (assuming, for simplicity that the query is returned as vector of strings):
template<typename T1, typename T2>
void populateTuple( boost::tuples::cons<T1, T2>& tupleRec, int index, const std::vector<std::string>& vals )
{
tupleRec.head = boost::lexical_cast<T1>( vals[index] );
populateTuple( tupleRec.tail, index+1, vals );
}
template<typename T>
void populateTuple( boost::tuples::cons<T, boost::tuples::null_type>& tupleRec, int index, const std::vector<std::string>& vals )
{
tupleRec.head = boost::lexical_cast<T>( vals[index] );
}
(Apologies - I've not run the above through a compiler, but am hoping it shows what I mean)
I'd love to be able to do something similar with Scala. I can recurse over a general Tuple type via the Product trait - and get the type of each element at run time using pattern matching (for the small number of column types I support). However I haven't found a way to assign Tuple elements via the product trait. And to be honest, I'm not convinced that this is a particularly nice or idiomatic way to do what I require anyway.
But something like:
val returnedFromQuery = List[String]( "Hello", "4", "6.0" )
val rowAsTuples = interpretListAsTuple[(String, Int, Float)]( returnedFromQuery )
Where rowAsTuples has type (String, Int, Float). Again, please excuse any syntax errors.
Anyone have any thoughts? Or alternative suggestions? In advance - I'm not interested in any higher-level SQL query libraries. I'm happy with sqlite4java but want to wrap it with a simple more abstract interface.
I think you should try using pattern-matching instead of interpretation. First, we need something that will pull out our types from strings using unapply:
object Stringy {
def unapply(s: String) = Some(s)
}
object Inty {
def unapply(s: String) = {
try { Some(s.toInt) }
catch { case nfe: NumberFormatException => None }
}
}
object Floaty {
def unapply(s: String) = {
try { Some(s.toFloat) }
catch { case nfe: NumberFormatException => None }
}
}
Now we can use them in pattern matches:
scala> List("Hello","4","6.0") match {
case Stringy(s) :: Inty(i) :: Floaty(f) :: Nil => Some((s,i,f))
case _ => None
}
res3: Option[(String, Int, Float)] = Some((Hello,4,6.0))
Note that if you do it this way, you (1) don't have to return tuples if you don't want to; (2) have access to all the parsed out variables right away; (3) have automatic error checking built in (with options).
Try HList from the MetaScala library.
I think you got problems with dynamic tuple arity, so you have to implement a method for each tuple arity, something like that:
def interpretListAsTuple2[A,B](s: List[String])(implicit s2a: String => A, s2b: String => B) = {
s.grouped(2).map { case x :: y => (x: A, y.head: B) }
}
def interpretListAsTuple3[A,B,C](s: List[String])(implicit s2a: String => A, s2b: String => B, s2c: String => C) = {
s.grouped(3).map { case x :: y :: z => (x: A, y: B, z.head: C) }
}
implicit def string2String(s: String) = s
implicit def string2Int (s: String) = s.toInt
implicit def string2Float (s: String) = s.toFloat
val returnedFromQuery = List( "Hello", "4", "6.0" )
interpretListAsTuple3[String,Int,Float](returnedFromQuery)
Sorry, this code doesn´t work, because of ambiguity of implicit conversions for String to float in scala´s Predef respectively LowPriorityImplicits. Perhaps someone can help and fix that. But the main idea should be clear, though. You only have to define the implicit conversions once for your data-types, then it works with all tuple-aritys.
EDIT:
You can use the above version to map a list with strings of several tuples. List("Hello", "4", "6.0","Hey","1", "2.3") If you want to handle only one tuple then use this:
def interpretListAsTuple3[A,B,C](s: List[String])(implicit s2a: String => A, s2b: String => B, s2c: String => C): (A,B,C) = {
s.grouped(3).map { case x :: y :: z => (x: A, y: B, z.head: C) }.next
}
Of course, arguments have to be well formed.
AFAIK, you can't get around type parameter arity. For evidence of that, take the fact that function and tuples have one definition for each arity, up to the arbitrary arity of 22.
If you nest type declarations, which is what existing implementations of HList do, then you can do something.
So I was asking this question because I wanted to write a simple wrapper around the sqlite4java sqlite interface. To enable code of the following form where the row type from a query could be specified in the prepared statement (I intend to add type checking to the parameters passed based on a similar method):
test("SQLite wrapper test")
{
val db = new SQLiteWrapper()
db.exec( "BEGIN" )
db.exec( "CREATE TABLE test( number INTEGER, value FLOAT, name TEXT )" )
val insStatement = db.prepare( "INSERT INTO test VALUES( ?, ?, ? )", HNil )
insStatement.exec( 1, 5.0, "Hello1" )
insStatement.exec( 2, 6.0, "Hello2" )
insStatement.exec( 3, 7.0, "Hello3" )
insStatement.exec( 4, 8.0, "Hello4" )
val getStatement = db.prepare( "SELECT * from test", Col[Int]::Col[Double]::Col[String]::HNil )
assert( getStatement.step() === true )
assert( _1(getStatement.row) === Some(1) )
assert( _2(getStatement.row) === Some(5.0) )
assert( _3(getStatement.row) === Some("Hello1") )
getStatement.reset()
db.exec( "ROLLBACK" )
}
And to enable this, using a variety of helpful SO suggestions I've come up with the code below. This is my first attempt at any form of generic programming in Scala - I've only been playing with the language for a week or two. So this is code is unlikely to be considered nice/good by the experienced Scala community. Any suggestions/feedback welcomed....
import java.io.File
import com.almworks.sqlite4java._
object SqliteWrapper
{
trait TypedCol[T]
{
var v : Option[T] = None
def assign( res : SQLiteStatement, index : Int )
}
sealed trait HList
{
def assign( res : SQLiteStatement, index : Int )
}
final case class HCons[H <: TypedCol[_], T <: HList]( var head : H, tail : T ) extends HList
{
def ::[T <: TypedCol[_]](v : T) = HCons(v, this)
def assign( res : SQLiteStatement, index : Int )
{
head.assign( res, index )
tail.assign( res, index+1 )
}
}
final class HNil extends HList
{
def ::[T <: TypedCol[_]](v : T) = HCons(v, this)
def assign( res : SQLiteStatement, index : Int )
{
}
}
type ::[H <: TypedCol[_], T <: HList] = HCons[H, T]
val HNil = new HNil()
final class IntCol extends TypedCol[Int]
{
def assign( res : SQLiteStatement, index : Int ) { v = Some( res.columnInt(index) ) }
}
final class DoubleCol extends TypedCol[Double]
{
def assign( res : SQLiteStatement, index : Int ) { v = Some( res.columnDouble(index) ) }
}
final class StringCol extends TypedCol[String]
{
def assign( res : SQLiteStatement, index : Int ) { v = Some( res.columnString(index) ) }
}
trait TypedColMaker[T]
{
def build() : TypedCol[T]
}
object TypedColMaker
{
implicit object IntColMaker extends TypedColMaker[Int]
{
def build() : TypedCol[Int] = new IntCol()
}
implicit object DoubleColMaker extends TypedColMaker[Double]
{
def build() : TypedCol[Double] = new DoubleCol()
}
implicit object StringColMaker extends TypedColMaker[String]
{
def build() : TypedCol[String] = new StringCol()
}
}
def Col[T : TypedColMaker]() = implicitly[TypedColMaker[T]].build()
// Hideousness. Improve as Scala metaprogramming ability improves
def _1[H <: TypedCol[_], T <: HList]( t : HCons[H, T] ) = t.head.v
def _2[H1 <: TypedCol[_], H2 <: TypedCol[_], T <: HList]( t : HCons[H1, HCons[H2, T]] ) = t.tail.head.v
def _3[H1 <: TypedCol[_], H2 <: TypedCol[_], H3 <: TypedCol[_], T <: HList]( t : HCons[H1, HCons[H2, HCons[H3, T]]] ) = t.tail.tail.head.v
def _4[H1 <: TypedCol[_], H2 <: TypedCol[_], H3 <: TypedCol[_], H4 <: TypedCol[_], T <: HList]( t : HCons[H1, HCons[H2, HCons[H3, HCons[H4, T]]]] ) = t.tail.tail.tail.head.v
def _5[H1 <: TypedCol[_], H2 <: TypedCol[_], H3 <: TypedCol[_], H4 <: TypedCol[_], H5 <: TypedCol[_], T <: HList]( t : HCons[H1, HCons[H2, HCons[H3, HCons[H4, HCons[H5, T]]]]] ) = t.tail.tail.tail.tail.head.v
def _6[H1 <: TypedCol[_], H2 <: TypedCol[_], H3 <: TypedCol[_], H4 <: TypedCol[_], H5 <: TypedCol[_], H6 <: TypedCol[_], T <: HList]( t : HCons[H1, HCons[H2, HCons[H3, HCons[H4, HCons[H5, HCons[H6, T]]]]]] ) = t.tail.tail.tail.tail.tail.head.v
def _7[H1 <: TypedCol[_], H2 <: TypedCol[_], H3 <: TypedCol[_], H4 <: TypedCol[_], H5 <: TypedCol[_], H6 <: TypedCol[_], H7 <: TypedCol[_], T <: HList]( t : HCons[H1, HCons[H2, HCons[H3, HCons[H4, HCons[H5, HCons[H6, HCons[H7, T]]]]]]] ) = t.tail.tail.tail.tail.tail.tail.head.v
def _8[H1 <: TypedCol[_], H2 <: TypedCol[_], H3 <: TypedCol[_], H4 <: TypedCol[_], H5 <: TypedCol[_], H6 <: TypedCol[_], H7 <: TypedCol[_], H8 <: TypedCol[_], T <: HList]( t : HCons[H1, HCons[H2, HCons[H3, HCons[H4, HCons[H5, HCons[H6, HCons[H7, HCons[H8, T]]]]]]]] ) = t.tail.tail.tail.tail.tail.tail.tail.head.v
final class DataWrapper[T <: HList]( var row : T )
{
def assign( res : SQLiteStatement ) { row.assign( res, 0 ) }
}
final class SQLiteWrapper( dbFile : File )
{
val conn = new SQLiteConnection( dbFile )
conn.open()
def exec( statement : String )
{
conn.exec( statement )
}
def prepare[T <: HList]( query : String, row : T ) =
{
new PreparedStatement(query, row)
}
// TODO: Parameterise with tuple type
// make applicable to for comprehensions (implement filter, map, flatMap)
final class PreparedStatement[T <: HList]( query : String, var row : T )
{
val statement = conn.prepare( query )
private def bindRec( index : Int, params : List[Any] )
{
println( "Value " + params.head )
// TODO: Does this need a pattern match?
params.head match
{
case v : Int => statement.bind( index, v )
case v : String => statement.bind( index, v )
case v : Double => statement.bind( index, v )
case _ => throw new ClassCastException( "Unsupported type in bind." )
}
if ( params.tail != Nil )
{
bindRec( index+1, params.tail )
}
}
def bind( args : Any* )
{
bindRec( 1, args.toList )
}
def exec( args : Any* )
{
bindRec( 1, args.toList )
step()
reset()
}
def reset()
{
statement.reset()
}
def step() : Boolean =
{
val success = statement.step()
row.assign( statement, 0 )
return success
}
}
}
}