See SQL From EntityFramework with Collection-like Queries - sql

I do not have 'full' the version of MS SQL (SQL Express 2008) so I do not have the profiler tool.
I want to see the SQL generated by my Entity Framework code, but all of the examples I find use the
var x = from u in table
select u;
type of syntax; But most of my queries are more like ..
var x = context.Users.Single(n => n.Name == "Steven");
type of syntax. What can I do to see the SQL generated, from this manner of coding? Any ideas?

check out this link... it has a tracing provider for EF4.
I'll try to give an example based on how I'm using this in a unit test.
Step 1: Register the provider
There are a couple ways to configure the provider. For my unit tests I configured the provider in code by calling the static method RegisterProvider();
[AssemblyInitialize()]
public static void AssemblyInit(TestContext context) {
EFTracingProviderConfiguration.RegisterProvider();
}
Step 2: Create a sub-class of your entity model to provide the tracing extensions
public partial class ExtendedNorthwindEntities : NorthwindEntities {
private TextWriter logOutput;
public ExtendedNorthwindEntities(string connectionString)
: base(EntityConnectionWrapperUtils.CreateEntityConnectionWithWrappers(
connectionString,
"EFTracingProvider")) {
}
#region Tracing Extensions
private EFTracingConnection TracingConnection {
get { return this.UnwrapConnection<EFTracingConnection>(); }
}
public event EventHandler<CommandExecutionEventArgs> CommandExecuting {
add { this.TracingConnection.CommandExecuting += value; }
remove { this.TracingConnection.CommandExecuting -= value; }
}
public event EventHandler<CommandExecutionEventArgs> CommandFinished {
add { this.TracingConnection.CommandFinished += value; }
remove { this.TracingConnection.CommandFinished -= value; }
}
public event EventHandler<CommandExecutionEventArgs> CommandFailed {
add { this.TracingConnection.CommandFailed += value; }
remove { this.TracingConnection.CommandFailed -= value; }
}
private void AppendToLog(object sender, CommandExecutionEventArgs e) {
if (this.logOutput != null) {
this.logOutput.WriteLine(e.ToTraceString().TrimEnd());
this.logOutput.WriteLine();
}
}
public TextWriter Log {
get { return this.logOutput; }
set {
if ((this.logOutput != null) != (value != null)) {
if (value == null) {
CommandExecuting -= AppendToLog;
}
else {
CommandExecuting += AppendToLog;
}
}
this.logOutput = value;
}
}
#endregion
}
Step 3: Attach to the Log property
var context = new ExtendedNorthwindEntities("name=\"NorthwindEntities\"");
context.Log = System.Console.Out;

Does Express Edition support extended events? If so this will capture statement and sp completed events in a similar way to Profiler.
Edit: I have changed it to use a memory target rather than a file target. Ideally uncomment the WHERE sections and replace with an appropriate user name to capture only events of interest or you can filter by spid with WHERE (([sqlserver].[session_id]=(56))) for example.
IF EXISTS(SELECT * FROM sys.server_event_sessions WHERE name='test_trace')
DROP EVENT SESSION [test_trace] ON SERVER;
CREATE EVENT SESSION [test_trace]
ON SERVER
ADD EVENT sqlserver.sp_statement_completed(
ACTION (package0.callstack, sqlserver.session_id, sqlserver.sql_text)
-- WHERE (([sqlserver].[username]='Domain\Username'))
),
ADD EVENT sqlserver.sql_statement_completed(
ACTION (package0.callstack, sqlserver.session_id, sqlserver.sql_text)
--WHERE (([sqlserver].[username]='Domain\Username'))
)
ADD TARGET package0.ring_buffer
WITH (MAX_MEMORY = 4096KB, EVENT_RETENTION_MODE = ALLOW_SINGLE_EVENT_LOSS,
MAX_DISPATCH_LATENCY = 1 SECONDS, MAX_EVENT_SIZE = 0KB,
MEMORY_PARTITION_MODE = NONE, TRACK_CAUSALITY = OFF, STARTUP_STATE = OFF)
ALTER EVENT SESSION [test_trace] ON SERVER STATE = START
And to review the results (Query generated using Adam Machanic's XE Code Generator)
DECLARE
#session_name VARCHAR(200) = 'test_trace'
SELECT
pivoted_data.*
FROM
(
SELECT MIN(event_name) AS event_name,
MIN(event_timestamp) AS event_timestamp,
unique_event_id,
CONVERT ( BIGINT, MIN (
CASE
WHEN d_name = 'cpu'
AND d_package IS NULL
THEN d_value
END ) ) AS [cpu],
CONVERT ( BIGINT, MIN (
CASE
WHEN d_name = 'duration'
AND d_package IS NULL
THEN d_value
END ) ) AS [duration],
CONVERT ( BIGINT, MIN (
CASE
WHEN d_name = 'object_id'
AND d_package IS NULL
THEN d_value
END ) ) AS [object_id],
CONVERT ( INT, MIN (
CASE
WHEN d_name = 'object_type'
AND d_package IS NULL
THEN d_value
END ) ) AS [object_type],
CONVERT ( DECIMAL(28,0), MIN (
CASE
WHEN d_name = 'reads'
AND d_package IS NULL
THEN d_value
END ) ) AS [reads],
CONVERT ( VARCHAR(MAX), MIN (
CASE
WHEN d_name = 'session_id'
AND d_package IS NOT NULL
THEN d_value
END ) ) AS [session_id],
CONVERT ( INT, MIN (
CASE
WHEN d_name = 'source_database_id'
AND d_package IS NULL
THEN d_value
END ) ) AS [source_database_id],
CAST((SELECT CONVERT ( VARCHAR(MAX), MIN (
CASE
WHEN d_name = 'sql_text'
AND d_package IS NOT NULL
THEN d_value
END ) ) AS [processing-instruction(x)] FOR XML PATH('') ) AS XML) AS [sql_text],
CONVERT ( DECIMAL(28,0), MIN (
CASE
WHEN d_name = 'writes'
AND d_package IS NULL
THEN d_value
END ) ) AS [writes]
FROM
(
SELECT
*,
CONVERT(VARCHAR(400), NULL) AS attach_activity_id
FROM
(
SELECT
event.value('(#name)[1]', 'VARCHAR(400)') as event_name,
event.value('(#timestamp)[1]', 'DATETIME') as event_timestamp,
DENSE_RANK() OVER (ORDER BY event) AS unique_event_id,
n.value('(#name)[1]', 'VARCHAR(400)') AS d_name,
n.value('(#package)[1]', 'VARCHAR(400)') AS d_package,
n.value('((value)[1]/text())[1]', 'VARCHAR(MAX)') AS d_value,
n.value('((text)[1]/text())[1]', 'VARCHAR(MAX)') AS d_text
FROM
(
SELECT
(
SELECT
CONVERT(xml, target_data)
FROM sys.dm_xe_session_targets st
JOIN sys.dm_xe_sessions s ON
s.address = st.event_session_address
WHERE
s.name = #session_name
AND st.target_name = 'ring_buffer'
) AS [x]
FOR XML PATH(''), TYPE
) AS the_xml(x)
CROSS APPLY x.nodes('//event') e (event)
CROSS APPLY event.nodes('*') AS q (n)
) AS data_data
) AS activity_data
GROUP BY
unique_event_id
) AS pivoted_data;

Related

how to use spark sql udaf to implement window counting with condition?

I have a table with columns: timestamp and id and condition, and I want to count the number of each id per interval such as 10 seconds.
If condition is true, the count++, otherwise return the previous value.
the udaf code like:
public class MyCount extends UserDefinedAggregateFunction {
#Override
public StructType inputSchema() {
return DataTypes.createStructType(
Arrays.asList(
DataTypes.createStructField("condition", DataTypes.BooleanType, true),
DataTypes.createStructField("timestamp", DataTypes.LongType, true),
DataTypes.createStructField("interval", DataTypes.IntegerType, true)
)
);
}
#Override
public StructType bufferSchema() {
return DataTypes.createStructType(
Arrays.asList(
DataTypes.createStructField("timestamp", DataTypes.LongType, true),
DataTypes.createStructField("count", DataTypes.LongType, true)
)
);
}
#Override
public DataType dataType() {
return DataTypes.LongType;
}
#Override
public boolean deterministic() {
return true;
}
#Override
public void initialize(MutableAggregationBuffer mutableAggregationBuffer) {
mutableAggregationBuffer.update(0, 0L);
mutableAggregationBuffer.update(1, 0L);
}
public void update(MutableAggregationBuffer mutableAggregationBuffer, Row row) {
long timestamp = mutableAggregationBuffer.getLong(0);
long count = mutableAggregationBuffer.getLong(1);
long event_time = row.getLong(1);
int interval = row.getInt(2);
if (event_time > timestamp + interval) {
timestamp = event_time - event_time % interval;
count = 0;
}
if (row.getBoolean(0)) {
count++;
}
mutableAggregationBuffer.update(0, timestamp);
mutableAggregationBuffer.update(1, count);
}
#Override
public void merge(MutableAggregationBuffer mutableAggregationBuffer, Row row) {
}
#Override
public Object evaluate(Row row) {
return row.getLong(1);
}
}
Then I sumbit a sql like:
select timestamp, id, MyCount(true, timestamp, 10) over(PARTITION BY id ORDER BY timestamp) as count from xxx.xxx
the result is:
timestamp id count
1642760594 0 1
1642760596 0 2
1642760599 0 3
1642760610 0 2 --duplicate
1642760610 0 2
1642760613 0 3
1642760594 1 1
1642760597 1 2
1642760600 1 1
1642760603 1 2
1642760606 1 4 --duplicate
1642760606 1 4
1642760608 1 5
When the timestamp is repeated, I get 1,2,4,4,5 instead of 1,2,3,4,5
How to fix it?
And another requestion is that when to execute the merge method of udaf? I empty implement it but it runs normally. I try to add the log in the method but I haven't seen this log. Is it really necessary?
There is a similar question: Apache Spark SQL UDAF over window showing odd behaviour with duplicate input
However, row_number() does not have such a problem. row_number() is a hive udaf, then I try to create a hive udaf. But I also have the problem...Why hive udaf row_number() terminate() returns 'ArrayList'? I create my udaf row_number2() by copying its code then I got list return?
Finally I solved it by spark aggregateWindowFunction:
case class Count(condition: Expression) extends AggregateWindowFunction with Logging {
override def prettyName: String = "myCount"
override def dataType: DataType = LongType
override def children: Seq[Expression] = Seq(condition)
private val zero = Literal(0L)
private val one = Literal(1L)
private val count = AttributeReference("count", LongType, nullable = false)()
private val increaseCount = If(condition, Add(count, one), count)
override val initialValues: Seq[Expression] = zero :: Nil
override val updateExpressions: Seq[Expression] = increaseCount :: Nil
override val evaluateExpression: Expression = count
override val aggBufferAttributes: Seq[AttributeReference] = count :: Nil
Then use spark_session.functionRegistry.registerFunction to register it.
"select myCount(true) over(partition by window(timestamp, '10 seconds'), id order by timestamp) as count from xxx"

Linq to SQL use case and Lag equivalent

I have this SQL which I am trying to convert to LINQ , how can this be converted?
Is there an equivalent of Lag at all?
I see there is a case statement not sure how to use it
SELECT
ah.AuthHist_ID,
ah.F_ID,
CASE WHEN ah.AuthPIFlg = 1 OR ah.AuthPINVFlg = 1 THEN 'True' ELSE 'False' end AS chkReqPI,
lag(CASE WHEN ah.AuthPIFlg = 1 OR ah.AuthPINVFlg = 1 THEN 'True' ELSE 'False' end, 1, null) OVER (ORDER BY ah.f_id, ah.AuthHist_ID) AS prevChkReqPI,
ah.Cr8Dt,
lag(ah.Cr8Dt, 1, null) OVER (ORDER BY ah.f_id, ah.AuthHist_ID) AS prevCr8Dt,
cu.UserName AS Cr8UserName,
lag(cu.UserName, 1, null) OVER (ORDER BY ah.f_id, ah.AuthHist_ID) AS prevCr8UserName,
fh.UpdtDt,
FROM AuthHist Ah
LEFT JOIN User cu
ON Ah.Cr8User_ID = cu.User_ID
WHERE Ah.F_ID = #fid
return (from a in DbContext.AuthHist
join c DbContext.User on a.UpdtUserId equals c.UserId
where a.FId == fId
select new AuthHistEntity()
{
FId = a.FId,
checkReqPI = a.AuthPIflg = 1 || a.AuthPINVflg = 1 :
});
I have this SQL which I am trying to convert to LINQ , how can this be
converted?
You can use linq to do this, but it will be a bit more complicated.
For the case when in linq, you can directly use the ternary operator(?:) instead.
As for the Lag() function, we need to replace this function with another sql writing as follow:
(This is just to facilitate the understanding of the linq statement below, you do not need to change your sql statement)
select kh.AuthHist_ID,kh.F_ID,
CASE WHEN kh.AuthPIFlg = 1 OR kh.AuthPINVFlg = 1 THEN 'True' ELSE 'False' end AS chkReqPI,
CASE WHEN (ch.AuthPIFlg is null or ch.AuthPINVFlg is null) then Null when (ch.AuthPIFlg = 1 OR ch.AuthPINVFlg = 1) THEN 'True' ELSE 'False' end AS prevChkReqPI ,
kh.Cr8Dt,
ch.Cr8Dt AS prevCr8Dt ,
kh.UserName AS Cr8UserName,
cu.UserName AS prevCr8UserName
from
( select -1+row_number() over (order by ah.f_id,ah.AuthHist_ID) as row1, * from
(select * from AuthHist a left join User u on a.Cr8User_ID = u.User_ID) Ah) kh
left join (select * from AuthHist a left join User u on a.Cr8User_ID = u.User_ID) cu ON kh.row1 = cu.AuthHist_ID
left join AuthHist ch on kh.row1 = ch.AuthHist_ID where kh.F_ID =#fid
Because the types of some fields are uncertain, I have create the AuthHistEntity class as follow, you can modify some details according to your needs.
public class AuthHistEntity
{
public int AuthHist_ID { get; set; }
public int F_ID { get; set; }
public string chkReqPI { get; set; }
public string prevChkReqPI { get; set; }
public string Cr8Dt { get; set; }
public string prevCr8Dt { get; set; }
public string Cr8UserName { get; set; }
public string prevCr8UserName { get; set; }
}
Here is the linq writing, you can try it:
var newUser = (from a in DbContext.AuthHist
join u in DbContext.User on a.Cr8User_ID equals u.User_ID
select new { a, u }).ToList();
var newAuthList = (from t in newUser
select new
{
row_1 = t.a.AuthHist_ID - 1,
data = t
}).ToList();
int fid = 1;
var result = (from dt in
(from ch in newAuthList
join ah in DbContext.AuthHist on ch.row_1 equals ah.AuthHist_ID
into ot from otnew in ot.DefaultIfEmpty()
select new { T1 = ch, T2 = otnew == null ? new AuthHist() : otnew
}).ToList()
join nu in newUser on dt.T1.row_1 equals nu.a.AuthHist_ID
into yG from otnew in yG.DefaultIfEmpty()
where dt.T1.data.a.F_ID == fid
select new AuthHistEntity()
{
AuthHist_ID = dt.T1.data.a.AuthHist_ID,
F_ID = dt.T1.data.a.F_ID,
chkReqPI = (dt.T1.data.a.AuthPIFlg == 1 || dt.T1.data.a.AuthPINVFlg == 1) ? "True" : "False",
prevChkReqPI = (dt.T2.AuthPIFlg == null || dt.T2.AuthPINVFlg == null) ? null : ((dt.T2.AuthPIFlg == 1 || dt.T2.AuthPINVFlg == 1) ? "True" : "Flase"),
Cr8Dt = dt.T1.data.a.Cr8Dt,
prevCr8Dt = dt.T2.Cr8Dt == null ? null : dt.T2.Cr8Dt,
Cr8UserName = dt.T1.data.u.UserName,
prevCr8UserName = otnew == null ? null : otnew.u.UserName
}).ToList();

SQL : recursive looping

Table is as follows:
I'm trying to get the result set to have groups of all person that intersects from the table, hence creating following groups in result set from attached table.
Person1, Person2, Person3, Person7, Person8
Person5, Person6, Person9
So far I have following query, but can't seem to get the results intersected on a table of rows and outputted as 1 column.
DECLARE #r VARCHAR(MAX), #n INT, #i INT
SELECT #i = 1,
#r = 'SELECT BOX, ' + CHAR(13),
#n = (SELECT TOP 1 COUNT( USERS )
FROM EXCHANGE
GROUP BY BOX
ORDER BY COUNT( USERS ) DESC ) ;
WHILE #i <= #n BEGIN
SET #r = #r +
CASE WHEN #i = 1
THEN 'MAX( CASE Seq WHEN ' + CAST( #i AS VARCHAR ) + '
THEN USERS
ELSE SPACE(0) END ) + ' + CHAR(13)
WHEN #i = #n
THEN 'MAX( CASE Seq WHEN ' + CAST( #i AS VARCHAR ) + '
THEN '', '' + USERS
ELSE SPACE(0) END ) ' + CHAR(13)
ELSE 'MAX( CASE Seq WHEN ' + CAST( #i AS VARCHAR ) + '
THEN '', '' + USERS
ELSE SPACE(0) END ) + ' + CHAR(13)
END ;
SET #i = #i + 1 ;
END
SET #r = #r + '
FROM ( SELECT BOX, USERS,
ROW_NUMBER() OVER ( PARTITION BY BOX ORDER BY USERS )
FROM EXCHANGE p ) D ( BOX, USERS, Seq )
GROUP BY BOX;'
EXEC( #r ) ;
This type of graph walking is a pain in SQL Server -- you have cycles. The problem is avoiding cycles. Because SQL Server doesn't have very good data types, you need to store the visited nodes as strings.
You can do all this in a recursive CTE. The idea is to follow all paths from a node without repeating any node. Keep the minimum node visited. Voila! That specifies the path:
with cte as (
select box, users,
convert(varchar(max), concat(',', box, ',', users, ',')) as path,
(case when box < users then box else users end) as min_node
from exchange
union all
select cte.box, e.users,
concat(cte.path, e.users, ','),
(case when min_node < e.users then min_node else e.users end)
from cte join
exchange e
on e.box = cte.users
where path not like '%,' + e.users + ',%'
)
select cte.box, min(cte.users), min(cte.path), min(cte.min_node) as grouping
from cte
group by cte.box;
Here is a db<>fiddle.
This assumes that the edges are symmetric, so if you have (a, b), you also have (b, a).
If this is not the case, it is easy to add a CTE that makes this the case:
select box, users
from exchange
union -- on purpose to remove duplicates
select users, box
from exchange;
using System;
using System.Collections.Generic;
using System.Data;
using System.Data.SqlClient;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace RecusriveGroup
{
public class FinalResult
{
public string GroupName { get; set; }
public string BoxName { get; set; }
public string UserName { get; set; }
}
class Program
{
static void Main(string[] args)
{
using(var con = new SqlConnection("Data Source=SQLServer;Initial Catalog=TESTDB;Integrated Security=SSPI"))
{
con.Open();
var cmd = new SqlCommand("select distinct Box from Exchange");
cmd.Connection = con;
var adapter = new SqlDataAdapter(cmd);
DataSet dsResult = new DataSet();
adapter.Fill(dsResult);
var finalResult = new List<FinalResult>();
var groupId = 0;
foreach (DataRow row in dsResult.Tables[0].Rows)
{
if(finalResult.Any(f => f.BoxName.Equals(row["Box"])))
{
continue;
}
groupId++;
RecursiveCall("Group" + groupId, row["Box"].ToString(), "", con, finalResult);
}
foreach(var result in finalResult)
{
var cmd1 = new SqlCommand("INSERT INTO FinalResult(Box, [User], [Group]) VALUES(#Box, #User, #Group)", con);
cmd1.Parameters.AddWithValue("#Box", result.BoxName);
cmd1.Parameters.AddWithValue("#User", result.UserName);
cmd1.Parameters.AddWithValue("#Group", result.GroupName);
cmd1.ExecuteNonQuery();
}
}
Console.ReadLine();
}
private static void RecursiveCall(string groupName, string boxName, string userName, SqlConnection sqlConnection, List<FinalResult> finalResult)
{
DataSet dsResult = new DataSet();
if (!string.IsNullOrEmpty(boxName) && !string.IsNullOrEmpty(userName))
{
var cmd = new SqlCommand("select Box, Users from Exchange WHERE Box = #BoxName OR Users = #UserName");
cmd.Parameters.AddWithValue("#BoxName", boxName);
cmd.Parameters.AddWithValue("#UserName", userName);
cmd.Connection = sqlConnection;
var adapter = new SqlDataAdapter(cmd);
adapter.Fill(dsResult);
}
else if(!string.IsNullOrEmpty(boxName))
{
var cmd = new SqlCommand("select Box, Users from Exchange WHERE Box = #BoxName");
cmd.Parameters.AddWithValue("#BoxName", boxName);
cmd.Connection = sqlConnection;
var adapter = new SqlDataAdapter(cmd);
adapter.Fill(dsResult);
}
else
{
var cmd = new SqlCommand("select Box, Users from Exchange WHERE Users = #UserName");
cmd.Parameters.AddWithValue("#UserName", userName);
cmd.Connection = sqlConnection;
var adapter = new SqlDataAdapter(cmd);
adapter.Fill(dsResult);
}
foreach (DataRow row in dsResult.Tables[0].Rows)
{
if (finalResult.Any(f => f.BoxName.Equals(row["Box"].ToString()) && f.UserName.Equals(row["Users"].ToString())))
{
continue;
}
finalResult.Add(new FinalResult() { GroupName = groupName, BoxName = row["Box"].ToString(), UserName = row["Users"].ToString() });
RecursiveCall(groupName, row["Box"].ToString(), row["Users"].ToString(), sqlConnection, finalResult);
}
}
}
}

Most efficient way to deal with ORA-01795:maximum number of expressions in a list is 1000 in hibernate

I have to perform a select on which I have more than 1000 elements via hibernate, and then I received the error "ORA-01795:maximum number of expressions in a list is 1000" when I'm using the Oracle brand.
SELECT * FROM matable WHERE column IN (?,?,...) (>1000 items)
I found many solutions :
Split the list with OR
where A in (a,b,c,d,e,f)
becomes
where (A in (a,b,c) OR a in (d,e,f)) ...
Create a table with UNION ALL
SELECT * FROM maintable
JOIN (
SELECT v1 a FROM DUAL UNION ALL
SELECT v2 a FROM DUAL UNION ALL
SELECT v3 a FROM DUAL UNION ALL
...
SELECT v2000 a FROM DUAL) tmp
on tmp.a = maintable.id
Using tuples to get rid of the limit
where (column,0) in ((1,0),(2,0),(3,0),(4,0), ... ,(1500,0))
Using a temporary table..
where A in SELECT item FROM my_temporary_table
References here and there and also there.
My question is the following : what is the best practice to deal with this issue? By best practice I mean the most performant, but not only for Oracle; if I use hibernate, I don't want to create and manage a different code for each brand of database (I'm concerned by Oracle, MS SQL and PostGre only).
My first reaction would have been to use a temporary table, but I don't know what has the most impact.
Use a temporary table and make the values primary keys on the table. This should allow very efficient optimizations for comparison. The most like is simply an index lookup, although if the table is very small, Oracle might choose some other method such as a table scan.
This method should be faster than 1,000 or conditions, in almost any database. Sometimes in is optimized in a similar way (using a binary tree to store the values). In such databases, the performance would be similar.
I fixed this issue with some changes in hibernate-core jar.
I made a helper class to split an expression in more joins like: ... t.column IN (: list_1) OR t.column IN (: list_2) ... , Then I changed AbstractQueryImpl.expandParameterList method from hibernate to call my method if the collection exceeds the limit.
My hibernate-core version is 3.6.10.Final, but it work fine and for 4.x versions - I tested it.
My code is tested for next cases:
where t.id in (:idList)
where (t.id in (:idList))
where ((t.id) in (:idList))
where 1=1 and t.id in (:idList)
where 1=1 and (t.id in (:idList))
where 1=1 and(t.id) in (:idList)
where 1=1 and((t.id) in (:idList))
where 1=1 and(t.id in (:idList))
where t.id not in (:idList)
where (t.id not in (:idList))
where ((t.id) not in (:idList))
AbstractQueryImpl.expandParameterList :
private String expandParameterList(String query, String name, TypedValue typedList, Map namedParamsCopy) {
Collection vals = (Collection) typedList.getValue();
Type type = typedList.getType();
boolean isJpaPositionalParam = parameterMetadata.getNamedParameterDescriptor( name ).isJpaStyle();
String paramPrefix = isJpaPositionalParam ? "?" : ParserHelper.HQL_VARIABLE_PREFIX;
String placeholder =
new StringBuffer( paramPrefix.length() + name.length() )
.append( paramPrefix ).append( name )
.toString();
if ( query == null ) {
return query;
}
int loc = query.indexOf( placeholder );
if ( loc < 0 ) {
return query;
}
String beforePlaceholder = query.substring( 0, loc );
String afterPlaceholder = query.substring( loc + placeholder.length() );
// check if placeholder is already immediately enclosed in parentheses
// (ignoring whitespace)
boolean isEnclosedInParens =
StringHelper.getLastNonWhitespaceCharacter( beforePlaceholder ) == '(' &&
StringHelper.getFirstNonWhitespaceCharacter( afterPlaceholder ) == ')';
if ( vals.size() == 1 && isEnclosedInParens ) {
// short-circuit for performance when only 1 value and the
// placeholder is already enclosed in parentheses...
namedParamsCopy.put( name, new TypedValue( type, vals.iterator().next(), session.getEntityMode() ) );
return query;
}
// *** changes by Vasile Bors for HHH-1123 ***
// case vals.size() > 1000
if ((vals.size() >= InExpressionExpander.MAX_ALLOWED_PER_INEXPR) && isEnclosedInParens) {
InExpressionExpander inExpressionExpander = new InExpressionExpander(beforePlaceholder, afterPlaceholder);
if(inExpressionExpander.isValidInOrNotInExpression()){
List<String> list = new ArrayList<String>( vals.size() );
Iterator iter = vals.iterator();
int i = 0;
String alias;
while ( iter.hasNext() ) {
alias = ( isJpaPositionalParam ? 'x' + name : name ) + i++ + '_';
namedParamsCopy.put( alias, new TypedValue( type, iter.next(), session.getEntityMode() ) );
list.add(ParserHelper.HQL_VARIABLE_PREFIX + alias );
}
String expandedExpression = inExpressionExpander.expandExpression(list);
if(expandedExpression != null){
return expandedExpression;
}
}
}
// *** end changes by Vasile Bors for HHH-1123 ***
StringBuffer list = new StringBuffer(16);
Iterator iter = vals.iterator();
int i = 0;
while (iter.hasNext()) {
String alias = (isJpaPositionalParam ? 'x' + name : name) + i++ + '_';
namedParamsCopy.put(alias, new TypedValue(type, iter.next(), session.getEntityMode()));
list.append(ParserHelper.HQL_VARIABLE_PREFIX).append(alias);
if (iter.hasNext()) {
list.append(", ");
}
}
return StringHelper.replace(
beforePlaceholder,
afterPlaceholder,
placeholder.toString(),
list.toString(),
true,
true
);
}
My helper class InExpressionExpander:
package org.hibernate.util;
import org.hibernate.QueryException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Iterator;
import java.util.List;
import java.util.Stack;
/**
* Utility class for expand Hql and Sql IN expressions with a parameter with more than IN expression limit size (HHH-1123).
* <br/>
* It work for expression with formats:
* <pre>
*
* where t.id in (:idList)
* where (t.id in (:idList))
* where ((t.id) in (:idList))
* where 1=1 and t.id in (:idList)
* where 1=1 and (t.id in (:idList))
* where 1=1 and(t.id) in (:idList)
* where 1=1 and((t.id) in (:idList))
* where 1=1 and(t.id in (:idList))
*
* where t.id not in (:idList)
* where (t.id not in (:idList))
* where ((t.id) not in (:idList))
* </pre>
* <p/>
* Example:
* <pre>
* select t.id from tableOrEntity t where t.id IN (:idList)
* </pre
*
* #author Vasile Bors
* #since 13/12/2015.
*/
public class InExpressionExpander {
private static final Logger log = LoggerFactory.getLogger(InExpressionExpander.class);
public static final int MAX_ALLOWED_PER_INEXPR = 1000;
private static final int MAX_PARAMS_PER_INEXPR = 500;
private Stack<String> stackExpr = new Stack<String>();
private StringBuilder toWalkQuery;
private final String beforePlaceholder;
private final String afterPlaceholder;
private boolean wasChecked = false;
private boolean isEnclosedInParens = false;
private boolean isInExpr = false;
private boolean isNotInExpr = false;
public InExpressionExpander(String beforePlaceholder, String afterPlaceholder) {
this.toWalkQuery = new StringBuilder(beforePlaceholder);
this.beforePlaceholder = beforePlaceholder;
this.afterPlaceholder = afterPlaceholder;
}
public boolean isValidInOrNotInExpression() {
if (!wasChecked) {
String lastExpr = extractLastExpression();
if ("(".equals(lastExpr)) {
isEnclosedInParens = true;
lastExpr = extractLastExpression();
}
isInExpr = "in".equalsIgnoreCase(lastExpr);
}
wasChecked = true;
return isInExpr;
}
public String expandExpression(List paramList) {
if (isValidInOrNotInExpression()) {
final String lastExpr = extractLastExpression(false);
if ("not".equalsIgnoreCase(lastExpr)) {
isNotInExpr = true;
extractLastExpression(); //extract "not" and consume it
}
extractColumnForInExpression();
StringBuilder exprPrefixBuilder = new StringBuilder();
for (int i = stackExpr.size() - 1; i > -1; i--) {
exprPrefixBuilder.append(stackExpr.get(i)).append(' ');
}
if (!isEnclosedInParens) {
exprPrefixBuilder.append('(');
}
String expandedExpression = expandInExpression(exprPrefixBuilder, paramList);
String beforeExpression = getBeforeExpression();
String afterExpression = getAfterExpression();
String expandedQuery = new StringBuilder(beforeExpression).append(expandedExpression)
.append(afterExpression)
.toString();
if (log.isDebugEnabled()) {
log.debug(
"Query was changed to prevent exception for maximum number of expression in a list. Expanded IN expression query:\n {}",
expandedExpression);
log.debug("Expanded query:\n {}", expandedQuery);
}
return expandedQuery;
}
log.error("Illegal call of InExpressionExpander.expandExpression() without IN expression.");
return null;
}
private String expandInExpression(StringBuilder exprPrefixBuilder, List values) {
String joinExpr = isNotInExpr ? ") and " : ") or ";
StringBuilder expr = new StringBuilder(16);
Iterator iter = values.iterator();
int i = 0;
boolean firstExpr = true;
while (iter.hasNext()) {
if (firstExpr || i % MAX_PARAMS_PER_INEXPR == 0) {
//close previous expression and start new expression
if (!firstExpr) {
expr.append(joinExpr);
} else {
firstExpr = false;
}
expr.append(exprPrefixBuilder);
} else {
expr.append(", ");
}
expr.append(iter.next());
i++;
}
expr.append(')');// close for last in expression
return expr.toString();
}
/**
* Method extract last expression parsed by space from toWalkQuery and remove it from toWalkQuery;<br/>
* If expression has brackets it will return al content between brackets and it will add additional space to adjust splitting by space.
*
* #return last expression from toWalkQuery
*/
private String extractLastExpression() {
return extractLastExpression(true);
}
/**
* Method extract last expression parsed by space from toWalkQuery, remove it from toWalkQuery if is consume = true;<br/>
* If expression has brackets it will return al content between brackets and it will add additional space to adjust splitting by space.
*
* #param consum if true the method will extract and remove last expression from toWalkQuery
* #return last expression from toWalkQuery
*/
private String extractLastExpression(final boolean consum) {
int lastIndex = this.toWalkQuery.length() - 1;
String lastExpr;
int exprSeparatorIndex = this.toWalkQuery.lastIndexOf(" ");
if (lastIndex == exprSeparatorIndex) { //remove last space from the end
this.toWalkQuery.delete(exprSeparatorIndex, this.toWalkQuery.length());
return extractLastExpression(consum);
} else {
lastExpr = this.toWalkQuery.substring(exprSeparatorIndex + 1, this.toWalkQuery.length());
if (lastExpr.length() > 1) {
if (lastExpr.endsWith(")")) {
//if parens are closed at the end we need to find where it is open
int opensParens = 0;
int closedParens = 0;
int startExprIndex = -1;
char c;
for (int i = lastExpr.length() - 1; i > -1; i--) {
c = lastExpr.charAt(i);
if (c == ')') {
closedParens++;
} else if (c == '(') {
opensParens++;
}
if (closedParens == opensParens) {
startExprIndex = i;
break;
}
}
if (startExprIndex > -1) {
lastExpr = lastExpr.substring(startExprIndex, lastExpr.length());
exprSeparatorIndex = exprSeparatorIndex + startExprIndex
+ 1; // +1 because separator is not space and don't must be deleted
}
} else if (lastExpr.contains("(")) {
int parentsIndex = exprSeparatorIndex + lastExpr.indexOf('(') + 1;
this.toWalkQuery.replace(parentsIndex, parentsIndex + 1, " ( ");
return extractLastExpression(consum);
}
}
if (consum) {
this.toWalkQuery.delete(exprSeparatorIndex, this.toWalkQuery.length());
}
}
if (consum) {
stackExpr.push(lastExpr);
}
return lastExpr;
}
private String extractColumnForInExpression() {
String column = extractLastExpression();
String beforeColumn = extractLastExpression(false);
long pointIndx = beforeColumn.lastIndexOf('.');
if (pointIndx > -1) {
if (pointIndx == (beforeColumn.length() - 1)) {
throw new QueryException(
"Invalid column format: " + beforeColumn + ' ' + column
+ " . Remove space from column!");
}
}
return column;
}
private String getBeforeExpression() {
return this.toWalkQuery + " (";
}
private String getAfterExpression() {
if (StringHelper.getFirstNonWhitespaceCharacter(afterPlaceholder) == ')') {
return afterPlaceholder;
}
return afterPlaceholder + ") ";
}
}
I am happy to receive any suggestions for improving this solution.

SQL aspnet_profile

any idea how I can get user FirstName and LastName from the aspnet_profile table based on UserID using SQL becasue I would like to use in Telerik Reporting as a user parameter.
Sample row (FirstName is George, LastName is Test):
UserID: 06b24b5c-9aa1-426e-b7e4-0771c5f85e85
PropertyName: MobilePhone:S:0:0:Initials:S:0:1:City:S:1:14:FirstName:S:15:6:PostalCode:S:21:7:‌​WorkPhone:S:28:12:LastName:S:40:5:Address1:S:45:17:Address2:S:62:0:Province:S:62:‌​2:Organization:S:64:4:ClinicId:S:68:1:Country:S:69:6:Fax:S:75:0:MSPNumber:S:75:0:‌​
PropertyValuesString: HEast HustonEASGeorgeT7D 1N8604-111-2222Test5555 Beddtvue AveDCHCNL2Canada
PropertyValuesBinary: <Binary data>
LastUpdateDate: 2010-01-02 22:22:03.947
If you insist on using SQL, I'm sure a large number of SUBSTRINGs and PATINDEXes will get you there but it won't be a clean solution.
Update: user373721 found a great resource and posted a comment about it, but it can be easily missed, so I decided to add it to the answer, too - How to get asp.net profile value from MS SQL database using T-SQL?
The built-in dbo.aspnet_Profile_GetProperties stored procedure returns the PropertyValuesString value that is later parsed in the ParseDataFromDB
function.
private void GetPropertyValuesFromDatabase(string userName, SettingsPropertyValueCollection svc)
{
if (HostingEnvironment.IsHosted && EtwTrace.IsTraceEnabled(4, 8))
{
EtwTrace.Trace(EtwTraceType.ETW_TYPE_PROFILE_BEGIN, HttpContext.Current.WorkerRequest);
}
HttpContext current = HttpContext.Current;
string[] names = null;
string values = null;
byte[] buffer = null;
if (current != null)
{
if (!current.Request.IsAuthenticated)
{
string anonymousID = current.Request.AnonymousID;
}
else
{
string name = current.User.Identity.Name;
}
}
try
{
SqlConnectionHolder connection = null;
SqlDataReader reader = null;
try
{
connection = SqlConnectionHelper.GetConnection(this._sqlConnectionString, true);
this.CheckSchemaVersion(connection.Connection);
SqlCommand command = new SqlCommand("dbo.aspnet_Profile_GetProperties", connection.Connection) {
CommandTimeout = this.CommandTimeout,
CommandType = CommandType.StoredProcedure
};
command.Parameters.Add(this.CreateInputParam("#ApplicationName", SqlDbType.NVarChar, this.ApplicationName));
command.Parameters.Add(this.CreateInputParam("#UserName", SqlDbType.NVarChar, userName));
command.Parameters.Add(this.CreateInputParam("#CurrentTimeUtc", SqlDbType.DateTime, DateTime.UtcNow));
reader = command.ExecuteReader(CommandBehavior.SingleRow);
if (reader.Read())
{
names = reader.GetString(0).Split(new char[] { ':' });
values = reader.GetString(1);
int length = (int) reader.GetBytes(2, 0L, null, 0, 0);
buffer = new byte[length];
reader.GetBytes(2, 0L, buffer, 0, length);
}
}
finally
{
if (connection != null)
{
connection.Close();
connection = null;
}
if (reader != null)
{
reader.Close();
}
}
ProfileModule.ParseDataFromDB(names, values, buffer, svc);
if (HostingEnvironment.IsHosted && EtwTrace.IsTraceEnabled(4, 8))
{
EtwTrace.Trace(EtwTraceType.ETW_TYPE_PROFILE_END, HttpContext.Current.WorkerRequest, userName);
}
}
catch
{
throw;
}
}
internal static void ParseDataFromDB(string[] names, string values, byte[] buf, SettingsPropertyValueCollection properties)
{
if (((names != null) && (values != null)) && ((buf != null) && (properties != null)))
{
try
{
for (int i = 0; i < (names.Length / 4); i++)
{
string str = names[i * 4];
SettingsPropertyValue value2 = properties[str];
if (value2 != null)
{
int startIndex = int.Parse(names[(i * 4) + 2], CultureInfo.InvariantCulture);
int length = int.Parse(names[(i * 4) + 3], CultureInfo.InvariantCulture);
if ((length == -1) && !value2.Property.PropertyType.IsValueType)
{
value2.PropertyValue = null;
value2.IsDirty = false;
value2.Deserialized = true;
}
if (((names[(i * 4) + 1] == "S") && (startIndex >= 0)) && ((length > 0) && (values.Length >= (startIndex + length))))
{
value2.SerializedValue = values.Substring(startIndex, length);
}
if (((names[(i * 4) + 1] == "B") && (startIndex >= 0)) && ((length > 0) && (buf.Length >= (startIndex + length))))
{
byte[] dst = new byte[length];
Buffer.BlockCopy(buf, startIndex, dst, 0, length);
value2.SerializedValue = dst;
}
}
}
}
catch
{
}
}
}
http://www.karpach.com/Get-asp-net-profile-value-MS-SQL-database-using-T-SQL.htm
this helped me tremendously!
following the steps in that link allowed me to fetch any particular data in the PropertyValueString from the aspnet_profile table.
copying and pasting-
First Function:
CREATE FUNCTION dbo.fn_GetElement
(
#ord AS INT,
#str AS VARCHAR(8000),
#delim AS VARCHAR(1) )
RETURNS INT
AS
BEGIN
-- If input is invalid, return null.
IF #str IS NULL
OR LEN(#str) = 0
OR #ord IS NULL
OR #ord < 1
-- #ord > [is the] expression that calculates the number of elements.
OR #ord > LEN(#str) - LEN(REPLACE(#str, #delim, '')) + 1
RETURN NULL
DECLARE #pos AS INT, #curord AS INT
SELECT #pos = 1, #curord = 1
-- Find next element's start position and increment index.
WHILE #curord < #ord
SELECT
#pos = CHARINDEX(#delim, #str, #pos) + 1,
#curord = #curord + 1
RETURN
CAST(SUBSTRING(#str, #pos, CHARINDEX(#delim, #str + #delim, #pos) - #pos) AS INT)
END
Second Function:
CREATE FUNCTION dbo.fn_GetProfileElement
(
#fieldName AS NVARCHAR(100),
#fields AS NVARCHAR(4000),
#values AS NVARCHAR(4000))
RETURNS NVARCHAR(4000)
AS
BEGIN
-- If input is invalid, return null.
IF #fieldName IS NULL
OR LEN(#fieldName) = 0
OR #fields IS NULL
OR LEN(#fields) = 0
OR #values IS NULL
OR LEN(#values) = 0
RETURN NULL
-- locate FieldName in Fields
DECLARE #fieldNameToken AS NVARCHAR(20)
DECLARE #fieldNameStart AS INTEGER,
#valueStart AS INTEGER,
#valueLength AS INTEGER
-- Only handle string type fields (:S:)
SET #fieldNameStart = CHARINDEX(#fieldName + ':S',#Fields,0)
-- If field is not found, return null
IF #fieldNameStart = 0 RETURN NULL
SET #fieldNameStart = #fieldNameStart + LEN(#fieldName) + 3
-- Get the field token which I've defined as the start of the
-- field offset to the end of the length
SET #fieldNameToken = SUBSTRING(#Fields,#fieldNameStart,LEN(#Fields)-#fieldNameStart)
-- Get the values for the offset and length
SET #valueStart = dbo.fn_getelement(1,#fieldNameToken,':')
SET #valueLength = dbo.fn_getelement(2,#fieldNameToken,':')
-- Check for sane values, 0 length means the profile item was
-- stored, just no data
IF #valueLength = 0 RETURN ''
-- Return the string
RETURN SUBSTRING(#values, #valueStart+1, #valueLength)
END
SQL Query can be modded to your needs
SELECT dbo.fn_GetProfileElement('FirstName',PropertyNames,PropertyValuesString)
, dbo.fn_GetProfileElement('LastName',PropertyNames,PropertyValuesString)
FROM aspnet_Profile
For those who are still looking for a method to parse the aspnet_Profile table using pure SQL. Here is what I use:
First you need a Tally table. If you do not know what this is, read this article by Jeff Moden: http://www.sqlservercentral.com/articles/T-SQL/62867/
For you to generate the tally table use this script:
SELECT TOP 11000 IDENTITY(INT,1,1) AS N INTO dbo.Tally FROM Master.dbo.SysColumns sc1, Master.dbo.SysColumns sc2
--===== Add a Primary Key to maximize performance
ALTER TABLE dbo.Tally ADD CONSTRAINT PK_Tally_N PRIMARY KEY CLUSTERED (N) WITH FILLFACTOR = 100
--===== Let the public use it
GRANT SELECT, REFERENCES ON dbo.Tally TO PUBLIC
Now on to parsing the ProfileData:
The process below is the fastest way I found to do this after lots of testing on my specific data. I have tested parsing the complete table in one go, but that runs slower than using the function below and parsing one user at a time with a CROSS APPLY.
So to call the function, use something like:
SELECT bla, bla
FROM aspnet_Users u CROSS APPY dbo.ProfileProperties(u.UserID)
The only thing you need to do is to update 3 things to contain the Profile Properties that you use:
1) the return table
2) the PIVOT statement, and
3) the insert statement copying the data from the PIVOT into the return table
Here is the function, Enjoy!
/** =============================================
** Author: Francois Grobler
** Create date: 2013-04-25
** Description: This function extracts all
** Profile Properties for a given UserId,
** and returns them as a table
** Change History:
** Date: Author: Change:
**
** ============================================= **/
CREATE FUNCTION dbo.ProfileProperties
(
#UserID UNIQUEIDENTIFIER
)
RETURNS #returnTable TABLE(
FirstName nvarchar(200)
, LastName nvarchar(200)
, PassportNumber nvarchar(100)
, PositionCode int
, CellNumber nvarchar(20)
, Telephone nvarchar(30)
, FaxNumber nvarchar(20)
, Email nvarchar(200)
, PersalNumber nvarchar(10)
, SouthAfricanIdentityNumber nchar(13)
, ContractNumber nvarchar(20)
, DepartmentName nvarchar(200)
, SiteName nvarchar(200)
, DepartmentCode int
, SiteCode int
, UserAccessCode int
, ApproverCode int
)
WITH SCHEMABINDING
AS
BEGIN
WITH Properties(PropertyNo, PropertyType, UserId, Value)
AS
(
SELECT (ROW_NUMBER() OVER(ORDER BY UserId) - 1) / 4 PropertyNo
, (ROW_NUMBER() OVER(PARTITION BY p.UserId ORDER BY UserId) - 1) % 4 PropertyType
, p.UserId
, SUBSTRING(':' + CONVERT(nvarchar(4000), p.PropertyNames), n + 1, CHARINDEX(':', ':' + CONVERT(nvarchar(4000), p.PropertyNames), n + 1) - n - 1) Value
FROM dbo.Tally, dbo.aspnet_Profile p
WHERE n < LEN(':' + CONVERT(nvarchar(4000), p.PropertyNames))
and SUBSTRING(':' + CONVERT(nvarchar(4000), p.PropertyNames), n, 1) = ':'
and p.UserId = #UserID
)
, FlatProperties(UserId, Property, ValueType, StartIndex, ValueLength)
AS
(
SELECT UserId
, MAX(CASE WHEN PropertyType = 0 THEN Value ELSE '' END) Property
, MAX(CASE WHEN PropertyType = 1 THEN Value ELSE '' END) ValueType
, MAX(CASE WHEN PropertyType = 2 THEN CONVERT(int, Value) + 1 ELSE 0 END) StartIndex
, MAX(CASE WHEN PropertyType = 3 THEN CONVERT(int, Value) ELSE 0 END) ValueLength
FROM
Properties
GROUP BY UserID, PropertyNo
)
, PropertyValues(UserID, PropertyName, PropertyValue)
AS
(
SELECT p.UserID, fp.Property
, CASE fp.ValueType
WHEN 'S' THEN SUBSTRING(p.PropertyValuesString, fp.StartIndex, fp.ValueLength)
ELSE SUBSTRING(p.PropertyValuesBinary, fp.StartIndex, fp.ValueLength) END Value
FROM dbo.aspnet_Profile p INNER JOIN flatProperties fp ON p.UserId = fp.UserId
WHERE p.UserId = #UserID
)
, PropertyTable
AS
(
SELECT
UserID
, pvt.[FirstName]
, pvt.[LastName]
, pvt.[PassportNumber]
, pvt.[PositionCode]
, pvt.[CellNumber]
, pvt.[Telephone]
, pvt.[FaxNumber]
, pvt.[Email]
, pvt.[PersalNumber]
, pvt.[SouthAfricanIdentityNumber]
, pvt.[ContractNumber]
, pvt.[DepartmentName]
, pvt.[SiteName]
, pvt.[DepartmentCode]
, pvt.[SiteCode]
, pvt.[UserCode] UserAccessCode
, pvt.[ApproverCode]
FROM PropertyValues
PIVOT (
MAX(PropertyValue) FOR PropertyName IN ([FirstName],[LastName],[PassportNumber],[PositionCode],[CellNumber],[Telephone],[FaxNumber],[Email],[PersalNumber],[SouthAfricanIdentityNumber],[ContractNumber],[DepartmentName],[SiteName],[DepartmentCode],[SiteCode],[UserCode],[ApproverCode])
) AS pvt
)
INSERT INTO #returnTable
(
FirstName
, LastName
, PassportNumber
, PositionCode
, CellNumber
, Telephone
, FaxNumber
, Email
, PersalNumber
, SouthAfricanIdentityNumber
, ContractNumber
, DepartmentName
, SiteName
, DepartmentCode
, SiteCode
, UserAccessCode
, ApproverCode
)
SELECT TOP 1
FirstName
, LastName
, PassportNumber
, PositionCode
, CellNumber
, Telephone
, FaxNumber
, Email
, PersalNumber
, SouthAfricanIdentityNumber
, ContractNumber
, DepartmentName
, SiteName
, DepartmentCode
, SiteCode
, UserAccessCode
, ApproverCode
FROM PropertyTable;
RETURN;
END
GO