I need to calculate an euclidean distance in a sqlite database.
Does anyone know how to calculate square roots in sqlite beside writing and loading a dynamic library for math functions?
I am close to resorting to the fast inverse square root algorithm in here http://en.wikipedia.org/wiki/Fast_inverse_square_root though it might to turn into more fun than I need right now.
And as a side note, it'd be great to figure out how to do power (which is the generalized question, and is cleaner coding than multiplying a number by itself).
Thanks,
Simone
Well, I have a semi-answer.
Yes it involves a 3rd party, but you don't have to write it yourself : did you check the last extension on this page ?
It includes several math functions, and amongst them is sqrt().
Warning: this answer is dependent on the coding language. In my case C#.
User defined SQLite functions was for me a pain to implement. Finally, after a long time of searching I was able to implement it in my C# code. Main function looks like this:
[SQLiteFunction(Arguments = 1, FuncType = FunctionType.Scalar, Name = "Sqrt")]
class Sqrt : SQLiteFunction
{
public override object Invoke(object[] args)
{
return Math.Sqrt(Double.Parse(args[0].ToString()));
}
}
Registration of custom function:
SQLiteFunction.RegisterFunction(typeof(Sqrt));
And using in select:
SQLiteCommand com = new SQLiteCommand("select sqrt(10.42)", connection);
You can download full example here: http://db.tt/qzeNXwso
Or, if you want only view code (or get through all parts of my code), I paste below full working example code for calculate square root in SQLite database, because is very hard to find any working code for this. To create and run this example do this 6 steps:
Create new project (my name is Sqrt)
Include SQLite reference to your project: Solution Explorer -> References (right click: Add reference) -> Assemblies - Extensions - System.Data.SQLite (check) -> OK
Open App.config and replace to this (without this step you maybe get Mixed mode assembly error):
<?xml version="1.0" encoding="utf-8" ?>
<configuration>
<startup useLegacyV2RuntimeActivationPolicy="true">
<supportedRuntime version="v4.0"/>
</startup>
</configuration>
Replace your Form1.Designer.cs with this code:
namespace Sqrt
{
partial class Form1
{
/// <summary>
/// Required designer variable.
/// </summary>
private System.ComponentModel.IContainer components = null;
/// <summary>
/// Clean up any resources being used.
/// </summary>
/// <param name="disposing">true if managed resources should be disposed; otherwise, false.</param>
protected override void Dispose(bool disposing)
{
if (disposing && (components != null))
{
components.Dispose();
}
base.Dispose(disposing);
}
#region Windows Form Designer generated code
/// <summary>
/// Required method for Designer support - do not modify
/// the contents of this method with the code editor.
/// </summary>
private void InitializeComponent()
{
this.txb_Input = new System.Windows.Forms.TextBox();
this.txb_Output = new System.Windows.Forms.TextBox();
this.label1 = new System.Windows.Forms.Label();
this.label2 = new System.Windows.Forms.Label();
this.btn_Calcualte = new System.Windows.Forms.Button();
this.SuspendLayout();
//
// txb_Input
//
this.txb_Input.Location = new System.Drawing.Point(131, 12);
this.txb_Input.Name = "txb_Input";
this.txb_Input.Size = new System.Drawing.Size(201, 20);
this.txb_Input.TabIndex = 0;
//
// txb_Output
//
this.txb_Output.BackColor = System.Drawing.Color.WhiteSmoke;
this.txb_Output.Location = new System.Drawing.Point(131, 38);
this.txb_Output.Name = "txb_Output";
this.txb_Output.ReadOnly = true;
this.txb_Output.Size = new System.Drawing.Size(201, 20);
this.txb_Output.TabIndex = 0;
//
// label1
//
this.label1.AutoSize = true;
this.label1.Location = new System.Drawing.Point(12, 15);
this.label1.Name = "label1";
this.label1.Size = new System.Drawing.Size(31, 13);
this.label1.TabIndex = 1;
this.label1.Text = "Input";
//
// label2
//
this.label2.AutoSize = true;
this.label2.Location = new System.Drawing.Point(12, 41);
this.label2.Name = "label2";
this.label2.Size = new System.Drawing.Size(39, 13);
this.label2.TabIndex = 1;
this.label2.Text = "Output";
//
// btn_Calcualte
//
this.btn_Calcualte.Location = new System.Drawing.Point(257, 64);
this.btn_Calcualte.Name = "btn_Calcualte";
this.btn_Calcualte.Size = new System.Drawing.Size(75, 23);
this.btn_Calcualte.TabIndex = 2;
this.btn_Calcualte.Text = "Calculate";
this.btn_Calcualte.UseVisualStyleBackColor = true;
//
// Form1
//
this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;
this.ClientSize = new System.Drawing.Size(344, 98);
this.Controls.Add(this.btn_Calcualte);
this.Controls.Add(this.label2);
this.Controls.Add(this.label1);
this.Controls.Add(this.txb_Output);
this.Controls.Add(this.txb_Input);
this.Name = "Form1";
this.Text = "Root square example";
this.ResumeLayout(false);
this.PerformLayout();
}
#endregion
private System.Windows.Forms.TextBox txb_Input;
private System.Windows.Forms.TextBox txb_Output;
private System.Windows.Forms.Label label1;
private System.Windows.Forms.Label label2;
private System.Windows.Forms.Button btn_Calcualte;
}
}
Open Form1.cs (code) and replace code with this:
using System;
using System.Data.SQLite;
using System.Windows.Forms;
namespace Sqrt
{
// definition of custom sqlite function
[SQLiteFunction(Arguments = 1, FuncType = FunctionType.Scalar, Name = "Sqrt")]
class Sqrt : SQLiteFunction
{
public override object Invoke(object[] args)
{
return Math.Sqrt(Double.Parse(args[0].ToString())); // return result of math sqrt function
}
}
public partial class Form1 : Form
{
public Form1()
{
InitializeComponent();
this.btn_Calcualte.Click += new System.EventHandler(this.btn_Calcualte_Click);
}
private void btn_Calcualte_Click(object sender, EventArgs e)
{
if (txb_Input.Text.Length == 0)
return;
try { SQLiteConnection.CreateFile(AppDomain.CurrentDomain.BaseDirectory + "test.s3db"); }
catch { }
SQLiteConnection con = new SQLiteConnection("Data Source=test.s3db");
SQLiteFunction.RegisterFunction(typeof(Sqrt)); // register custom function
con.Open();
SQLiteCommand com = new SQLiteCommand("select sqrt(" + txb_Input.Text.Replace(',', '.') + ")", con); // select result
string res = com.ExecuteScalar().ToString();
txb_Output.Text = res;
}
}
}
Run, try and enjoy.
This is an approximation of sqrt for numbers under 10000. It can be extended for arbitrary numbers, and can be extended to arbitrary precision as needed. This kind of tabular interpolation is what happens in most fast implementations anyway:
case when weight >= 1 and weight<=10 then 1+0.240253073*(weight-1)
when weight>=10 and weight<=100 then 3.16227766+0.075974693*(weight-10)
when weight>=100 and weight<=1000 then 10+0.024025307*(weight-100)
else 31.6227766+0.007597469 *(weight-1000) end
And there's the curious fact that each factor you use in such a power-of-10 square root interpolation table is 0.316227766 times the previous one - so you can make this work for an arbitrarily large number, or even stuff a table full of these values to make it work for any number. (Could that lead to some compression here?)
Or this cute one for log10 of integers, using the length function (an interpolation table might work better here, but I like that log10 and length() are similar, and that this works for any integer - no interpolation needed.
((length(x)+length(x*2)+length(x*3)
+length(x*4)+length(x*5))/5.0)-1.0
A better math head than I can probably come up with better and denser approximations. Considering that most sqrt functions in c use approximations anyway - this is a pretty good solution.
This is the only native way of doing it.
As far as I know - you can't do that using only core functions.
Here is the list of native functions Core functions and a list of aggregate functions Aggregate functions.
To solve your problem, you can write your own UDF (user defined function) as illustrated HERE
Only if math functions are not available... and really only in desperation because this isn't gonna be fast...
-- bisect to find the square root to any tolerance desired
with
input(n) as (select 500), --input
sqrt(lo, hi, guess, n, i) as (
select 1, n, n/2, n, 0 from input
union all
select case when guess*guess < n then guess else lo end,
case when guess*guess < n then hi else guess end,
case when guess*guess < n then (hi+guess)/2.0 else (lo+guess)/2.0 end,
n ,
i +1 from sqrt
where abs(guess*guess - n) > 0.0001), -- tolerance
sqrt_out(x, n) as (select guess, n from sqrt order by sqrt.i desc limit 1)
select * from sqrt_out
2021-03-12 (3.35.0)
Added built-in SQL math functions(). (Requires the -DSQLITE_ENABLE_MATH_FUNCTIONS compile-time option.)
Built-In Mathematical SQL Functions
sqrt(X) Return the square root of X. NULL is returned if X is negative.
Related
I have custom extractor, and I'm trying to log some messages from it.
I've tried obvious things like Console.WriteLine, but cannot find where output is. However, I found some system logs in adl://<my_DLS>.azuredatalakestore.net/system/jobservice/jobs/Usql/.../<my_job_id>/.
How can I log something? Is it possible to specify log file somewhere on Data Lake Store or Blob Storage Account?
A recent release of U-SQL has added diagnostic logging for UDOs. See the release notes here.
// Enable the diagnostics preview feature
SET ##FeaturePreviews = "DIAGNOSTICS:ON";
// Extract as one column
#input =
EXTRACT col string
FROM "/input/input42.txt"
USING new Utilities.MyExtractor();
#output =
SELECT *
FROM #input;
// Output the file
OUTPUT #output
TO "/output/output.txt"
USING Outputters.Tsv(quoting : false);
This was my diagnostic line from the UDO:
Microsoft.Analytics.Diagnostics.DiagnosticStream.WriteLine(System.String.Format("Concatenations done: {0}", i));
This is the whole UDO:
using System.Collections.Generic;
using System.IO;
using System.Text;
using Microsoft.Analytics.Interfaces;
namespace Utilities
{
[SqlUserDefinedExtractor(AtomicFileProcessing = true)]
public class MyExtractor : IExtractor
{
//Contains the row
private readonly Encoding _encoding;
private readonly byte[] _row_delim;
private readonly char _col_delim;
public MyExtractor()
{
_encoding = Encoding.UTF8;
_row_delim = _encoding.GetBytes("\n\n");
_col_delim = '|';
}
public override IEnumerable<IRow> Extract(IUnstructuredReader input, IUpdatableRow output)
{
string s = string.Empty;
string x = string.Empty;
int i = 0;
foreach (var current in input.Split(_row_delim))
{
using (System.IO.StreamReader streamReader = new StreamReader(current, this._encoding))
{
while ((s = streamReader.ReadLine()) != null)
{
//Strip any line feeds
//s = s.Replace("/n", "");
// Concatenate the lines
x += s;
i += 1;
}
Microsoft.Analytics.Diagnostics.DiagnosticStream.WriteLine(System.String.Format("Concatenations done: {0}", i));
//Create the output
output.Set<string>(0, x);
yield return output.AsReadOnly();
// Reset
x = string.Empty;
}
}
}
}
}
And these were my results found in the following directory:
/system/jobservice/jobs/Usql/2017/10/20.../diagnosticstreams
good question. I have been asking myself the same thing. This is theoretical, but I think it would work (I'll updated if I find differently).
One very hacky way is that you could insert rows into a table with your log messages as a string column. Then you can select those out and filter based on some log_producer_id column. You also get the benefit of logging if part of the script works, but later parts do not assuming the failure does not roll back. Table can be dumped at end as well to file.
For the error cases, you can use the Job Manager in ADLA to open the job graph and then view the job output. The errors often have detailed information for data-related errors (e.g. row number in file with error and a octal/hex/ascii dump of the row with issue marked with ###).
Hope this helps,
J
ps. This isn't a comment or an answer really, since I don't have working code. Please provide feedback if the above ideas are wrong.
We are using XGboost4J for ML predictions. We developed predictor using restful webservice so that within platform various components can call ML predictor. e.g from product titles and description finding out product category tree.
Just depicting code in fundamental way we implemented.
// This is done in
initialize method, for every model there is one singleton Booster object loaded.
Class Predictor{
private Booster xgboost;
//init call from Serivice initialization while injecting Predictor
public void init(final String modelFile, final Integer numThreads){
if (!(new File(modelFile).exists())) {
throw new IOException("Modelfile " + modelFile + " does not exist");
}
// we use a util class Params to handle parameters as example
final Iterable<Entry<String, Object>> param = new Params() {
{
put("nthread", numThreads);
}
};
xgboost = new Booster(param, modelFile);
}
//Predict method
public String predict(final String predictionString){
final String dummyLabel = "-1";
final String x_n = dummyLabel + "\t" + x_n_libsvm_idxStr;
final DataLoader.CSRSparseData spData = XGboostSparseData.format(x_n);
final DMatrix x_n_dmatrix = new DMatrix(spData.rowHeaders,
spData.colIndex, spData.data, DMatrix.SparseType.CSR);
final float[][] predict = xgboost.predict(x_n_dmatrix);
// Then there is conversion logic of predict to predicted model result which returns predictions
String prediction = getPrediction(predict);
return prediction
}
}
Above predictor class is singleton injected in webservices Services class
so for every services call thread call's
service.predict(predictionString);
There is problem in tomcat container when multiple concurrent threads calls predict method Boosters method is synchronized
private synchronized float[][] pred(DMatrix data, boolean outPutMargin, long treeLimit, boolean predLeaf) throws XGBoostError {
byte optionMask = 0;
if(outPutMargin) {
optionMask = 1;
}
if(predLeaf) {
optionMask = 2;
}
float[][] rawPredicts = new float[1][];
ErrorHandle.checkCall(XgboostJNI.XGBoosterPredict(this.handle, data.getHandle(), optionMask, treeLimit, rawPredicts));
int row = (int)data.rowNum();
int col = rawPredicts[0].length / row;
float[][] predicts = new float[row][col];
for(int i = 0; i < rawPredicts[0].length; ++i) {
int r = i / col;
int c = i % col;
predicts[r][c] = rawPredicts[0][i];
}
return predicts;
}
This created thread waits and locking because of synchronized block and this is resulting webservices not scalable.
We tried removing synchronized from XGboost4J source code and compiled jar but it crashes within first 1-2 mins. Heap dump showing its crashing at below line while doing native call to XgboostJNI
ErrorHandle.checkCall(XgboostJNI.XGBoosterPredict(this.handle, data.getHandle(), optionMask, treeLimit, rawPredicts));
Anyone knows better way of implementing Xgboost4J for highly scalable webservices approach using Java?
You could use PMML (https://github.com/jpmml/jpmml-xgboost), referring to https://github.com/jpmml/jpmml-xgboost/issues/7#issuecomment-250965282
I have an observable sequence that should be filtered using the Where operator based on some criteria, and should populate a list with those filtered elements.
I want to change the filtering criteria on the fly, so when this happens, the list should be cleared, the previous elements of the observable sequence should be evaluated for the new criteria, as well as new elements produced by the sequence.
The list should be repopulated with past elements which now passes the new criteria, continuing with new elements (filtered as well). I don't care that new elements might be delayed, while previous elements are re-evaluated, but i do care about order.
Is this something that can be done with Reactive Extensions?
Here's a nice extension method that does what you need:
public static IObservable<T> Refilterable<T>(
this IObservable<T> source, IObservable<Func<T, bool>> filters)
{
return
Observable
.Create<T>(o =>
{
var replay = new ReplaySubject<T>();
var replaySubscription = source.Subscribe(replay);
var query = filters.Select(f => replay.Where(f)).Switch();
var querySubscription = query.Subscribe(o);
return new CompositeDisposable(replaySubscription, querySubscription);
});
}
I tested this with this code:
var source = new Subject<int>();
var filters = new Subject<Func<int, bool>>();
var subscription = source.Refilterable(filters).Subscribe(x => Console.WriteLine(x));
source.OnNext(1);
source.OnNext(2);
source.OnNext(3);
filters.OnNext(x => x % 2 == 0);
source.OnNext(4);
source.OnNext(5);
filters.OnNext(x => x % 2 == 1);
source.OnNext(6);
filters.OnNext(x => x % 3 == 0);
source.OnNext(7);
filters.OnNext(x => x % 2 == 1);
subscription.Dispose();
filters.OnNext(x => x % 2 == 0);
I got this output:
2
4
1
3
5
3
6
1
3
5
7
Which seems to be what you're after.
I just noticed the requirement to produce lists. Here's an update:
public static IObservable<IList<T>> Refilterable<T>(this IObservable<T> source, IObservable<Func<T, bool>> filters)
{
return
Observable
.Create<IList<T>>(o =>
{
var replay = new ReplaySubject<T>();
var replaySubscription = source.Subscribe(replay);
var query =
filters
.Select(f =>
replay
.Synchronize()
.Where(f)
.Scan(new List<T>(), (a, x) =>
{
a.Add(x);
return a;
}))
.Switch();
var querySubscription = query.Subscribe(o);
return new CompositeDisposable(replaySubscription, querySubscription);
});
}
The only other thing I noticed was the VB.NET tag. I'll see if I can convert later if need be.
This should be the right VB:
<System.Runtime.CompilerServices.Extension> _
Public Shared Function Refilterable(Of T)(source As IObservable(Of T), filters As IObservable(Of Func(Of T, Boolean))) As IObservable(Of IList(Of T))
Return Observable.Create(Of IList(Of T))( _
Function(o)
Dim replay = New ReplaySubject(Of T)()
Dim replaySubscription = source.Subscribe(replay)
Dim query = filters.[Select](Function(f) replay.Synchronize().Where(f).Scan(New List(Of T)(), _
Function(a, x)
a.Add(x)
Return a
End Function)).Switch()
Dim querySubscription = query.Subscribe(o)
Return New CompositeDisposable(replaySubscription, querySubscription)
End Function)
End Function
Unless I missed something crucial, the specified requirements eliminate the need for immutable collections and simplify the implementation because you need to buffer all emitted values.
private List<T> values = new List<T>();
private IObservable<T> _valueSource;
public List<T> ValidValues => values.Where(MatchesCriteria).ToList();
private void StartSubscriptions()
{
var addNewValuesSub = _valueSource.Subscribe(values.Add); //todo disposing
}
If you believe that IEnumerable.Where is too slow and we know all possible criterias beforehand. We could GroupBy separate values to their respective Observables / data structures. It would look something like this.
_valueSource.GroupBy(CriteriaSelector)
.Subscribe(i => UpdateDataStructure(i.Key(), i.Latest()) );
IObservable is not good for buffering lots of values which should later be accessed by some criteria. That's a job for IEnumerable.
Finally, if you think memory usage will be an issue consider refactoring values as a memory object caching system.
I have two initial reactions
That is just a .Replay() that is resubscribed to when the filter condition changes
That also sounds like an unbounded buffer :-/
To address my first thought, you could just have this code
public class Foo
{
private readonly IDisposable _replayConnection;
private readonly IConnectableObservable<int> _replaySource;
private readonly SerialDisposable _subscription = new SerialDisposable();
private readonly List<int> _values = new List<int>();
//the Ctor or some initialise method...
public Foo(IObservable<int> source)
{
_replaySource = source.Replay();
_replayConnection = _replaySource.Connect()
}
public void SetFilter(Func<int, bool> predicate)
{
//Not thread safe. If required, then a scheduler can solve that.
_values.Clear();
_subscription.Disposable = _replaySource.Where(predicate)
.Subscribe(value => _values.Add(value),
ex => {/*Do something here to handle errors*/},
() => {/*Do something here if you want to cater for completion of the sequence*/},
}
}
However, this just makes me more concerned about point 2. If you are expecting millions of values, then if they are just ints, then it is about 3MB/1mil items of memory you will be using. If they are ints, then copy by value semantics, you will get copies in the Replay buffer and in your final list (IIRC). If this kind of memory pressure is ok, then I think the Replay code above will be fine. Also note that this Replay usage will throw if you try to buffer more than int.MaxValue values.
I know this question has been asked a bunch of times, but none of the answers (or at least what i took away from them) was a help to my particiular problem.
I want to dynamically change a part of the variable path, so i don't have to repeat the same code x-times with just two characters changing.
Here's what i got:
In the beginning of my script, i'm setting the reference to PlayerData scripts, attached to the GameManager object like this:
var P1 : P1_Data;
var P2 : P2_Data;
function Start(){
P1 = GameObject.Find("GameManager").GetComponent.<P1_Data>();
P2 = GameObject.Find("GameManager").GetComponent.<P2_Data>();
}
Later, i want to access these scripts using the currentPlayer variable to dynamically adjust the path:
var currentPlayer : String = "P1"; //this actually happens along with some other stuff in the SwitchPlayers function, i just put it here for better understanding
if (currentPlayer.PlayerEnergy >= value){
// do stuff
}
As i was afraid, i got an error saying, that PlayerEnergy was not a part of UnityEngine.String.
So how do I get unity to read "currentPlayer" as part of the variable path?
Maybe some parse function I haven't found?
Or am I going down an entirely wrong road here?
Cheers
PS: I also tried putting the P1 and P2 variables into an array and access them like this:
if (PlayerData[CurrentPlayerInt].PlayerEnergy >= value){
// do stuff
}
to no success.
First of all,
var currentPlayer : String = "P1"
here P1 is just string, not the previous P1/P2 which are referenced to two scripts. So, if you want, you can change
currentPlayer.PlayerEnergy >= value
to
P1.PlayerEnergy >= value
or,
P2.PlayerEnergy >= value
But if you just want one function for them, like
currentPlayer.PlayerEnergy >= value
Then you have to first set currentPlayer to P1/P2 which I assume you are trying to do. You must have some codes that can verify which player is selected. Then, maybe this can help -
var playerSelected: int = 0;
var currentPlayerEnergy: int = 0;
.....
//Use your codes to verify which player is selected and then,
if (playerSelected == 1) {
currentPlayerEnergy = P1.PlayerEnergy;
} else if (playerSelected == 2) {
currentPlayerEnergy = P2.PlayerEnergy;
}
//Now use your favorite function
if (currentPlayerEnergy >= value) {
//Do stuff
}
As there was no reply providing the answer I needed, I'll share the solution that did the trick for me, provided by a fellow student.
Instead of having the PlayerData scripts pre-written, I generate them using a public class function in a Playermanager script. This generates the Playerdata as attached scripts, saved into an array.
I can then access them through Playermanager.Playerlist[Playernumber].targetvariable.
Which is what I wanted to do, only with the Playerdata being attached to a script instead of a gameobject. And it works great!
Here's the full code of my Playermanager Script:
//initialise max players
public var maxplayers : int = 2;
// Initialise Playerlist
static var Players = new List.<PlayerData>();
function Start () {
for (var i : int = 0; i < maxplayers; i++){
var Player = new PlayerData();
Players.Add(Player);
Players[i].PlayerName = "Player " + i;
}
DontDestroyOnLoad (transform.gameObject);
}
public class PlayerData {
public var PlayerName : String;
public var PlayerEnergy : int = 15;
public var Fleet : List.<GameObject> = new List.<GameObject>();
}
As you see, you can put any type of variable in this class.
I hope this helps some of you who have the same problem.
cheers,
Tux
I'm working with Actionscript 2 (not ready to upgrade yet, although it's irreverent to the problem) but I'm having trouble with OOP and classes.
I've got a "Tool" class, written like so:
class com.Tool {
public var self:MovieClip;
private static var Type:String;
function Tool(T:String, X:Number, Y:Number) {
Type = T;
self = _root.createEmptyMovieClip("obj"+_root.getNextHighestDepth(), _root.getNextHighestDepth());
self._x = X;
self._y = Y;
self.width = 36;
self.height = 36;
self.onRollOver = function() {
trace(Type);
}
}
}
I create 3 of them in the main script like so:
var toolPan:Tool = new Tool("pan", 0, 0);
var toolSquare:Tool = new Tool("square", 0, 38);
var toolLine:Tool = new Tool("line", 0, 76);
It all works great, except the onRollOver. It's supposed to output the unique "Type" string, but it always outputs "line" (the last Type Tool created) regardless which one I roll over.
Needless to say, I'm still a beginner to all this. But it seems like they're all sharing the same variable :/ How do I make these variables unique to each object created?
Thank you very much!
It's because it's type static, so the value is shared by all instances of that class. Remove it and it should work.
private var Type:String;