When I try to maximize my objective function using nloptr, it is terminating on the initial values that I have set, which I think is very unlikely to return the maximum.
For some background, I am trying to allocate 80M of investment (x) between 114 products in a way that maximizes revenue (y). Predicted revenue for each product and investment level is based off a two-part model i.e. it is the result of multiplying the predicted probability of any revenue being generated, by the predicted level of revenue, conditional on revenue being generated. Each line of the objective function represents the fitted model for each product.
library(nloptr)
# objective function
eval_f0 <- function(x){
return(-(plogis(0.872 + 0.0000471*x[1]-1.04483)*exp(8.459+0.468*log(x[1])+1.07743) +
plogis(0.872 + 0.0000471*x[2]-1.04483)*exp(8.459+0.468*log(x[2])+1.07743) +
plogis(0.872 + 0.0000471*x[3]-1.04483)*exp(8.459+0.468*log(x[3])+1.07743) +
plogis(0.872 + 0.0000471*x[4]-1.04483)*exp(8.459+0.468*log(x[4])+1.07743) +
plogis(0.872 + 0.0000471*x[5]-1.04483)*exp(8.459+0.468*log(x[5])+1.07743) +
plogis(0.872 + 0.0000471*x[6]-1.04483)*exp(8.459+0.468*log(x[6])+1.07743) +
plogis(0.872 + 0.0000471*x[7]-1.04483)*exp(8.459+0.468*log(x[7])+1.07743) +
plogis(0.872 + 0.0000471*x[8]-1.04483)*exp(8.459+0.468*log(x[8])+1.07743) +
plogis(0.872 + 0.0000471*x[9]-1.04483)*exp(8.459+0.468*log(x[9])+1.07743) +
plogis(0.872 + 0.0000471*x[10]-1.04483)*exp(8.459+0.468*log(x[10])+1.07743) +
plogis(0.872 + 0.0000471*x[11]-1.04483)*exp(8.459+0.468*log(x[11])+1.07743) +
plogis(0.872 + 0.0000471*x[12]-1.04483)*exp(8.459+0.468*log(x[12])+1.07743) +
plogis(0.872 + 0.0000471*x[13]-1.04483)*exp(8.459+0.468*log(x[13])+1.07743) +
plogis(0.872 + 0.0000471*x[14]-1.04483)*exp(8.459+0.468*log(x[14])+1.07743) +
plogis(0.872 + 0.0000471*x[15]-1.04483)*exp(8.459+0.468*log(x[15])+1.07743) +
plogis(0.872 + 0.0000471*x[16]-1.04483)*exp(8.459+0.468*log(x[16])+1.07743) +
plogis(0.872 + 0.0000471*x[17]-1.04483)*exp(8.459+0.468*log(x[17])+1.07743) +
plogis(0.872 + 0.0000471*x[18]-1.04483)*exp(8.459+0.468*log(x[18])+1.07743) +
plogis(0.872 + 0.0000471*x[19]-1.04483)*exp(8.459+0.468*log(x[19])+1.07743) +
plogis(0.872 + 0.0000471*x[20]-1.04483)*exp(8.459+0.468*log(x[20])+1.07743) +
plogis(0.872 + 0.0000471*x[21]-1.04483)*exp(8.459+0.468*log(x[21])+1.07743) +
plogis(0.872 + 0.0000471*x[22]-1.04483)*exp(8.459+0.468*log(x[22])+1.07743) +
plogis(0.872 + 0.0000471*x[23]-1.04483)*exp(8.459+0.468*log(x[23])+1.07743) +
plogis(0.872 + 0.0000471*x[24]-1.04483)*exp(8.459+0.468*log(x[24])+1.07743) +
plogis(0.872 + 0.0000471*x[25]-1.04483)*exp(8.459+0.468*log(x[25])+1.07743) +
plogis(0.872 + 0.0000471*x[26]-1.04483)*exp(8.459+0.468*log(x[26])+1.07743) +
plogis(0.872 + 0.0000471*x[27]-1.04483)*exp(8.459+0.468*log(x[27])+1.07743) +
plogis(0.872 + 0.0000471*x[28]-1.04483)*exp(8.459+0.468*log(x[28])+1.07743) +
plogis(0.872 + 0.0000471*x[29]-1.04483)*exp(8.459+0.468*log(x[29])+1.07743) +
plogis(0.872 + 0.0000471*x[30]-1.04483)*exp(8.459+0.468*log(x[30])+1.07743) +
plogis(0.872 + 0.0000471*x[31]-1.04483)*exp(8.459+0.468*log(x[31])+1.07743) +
plogis(0.872 + 0.0000471*x[32]-1.04483)*exp(8.459+0.468*log(x[32])+1.07743) +
plogis(0.872 + 0.0000471*x[33]-1.04483)*exp(8.459+0.468*log(x[33])+1.07743) +
plogis(0.872 + 0.0000471*x[34]-1.04483)*exp(8.459+0.468*log(x[34])+1.07743) +
plogis(0.872 + 0.0000471*x[35]-1.04483+0.289)*exp(8.459+0.468*log(x[35])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[36]-1.04483+0.289)*exp(8.459+0.468*log(x[36])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[37]-1.04483+0.289)*exp(8.459+0.468*log(x[37])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[38]-1.04483+0.289)*exp(8.459+0.468*log(x[38])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[39]-1.04483+0.289)*exp(8.459+0.468*log(x[39])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[40]-1.04483+0.289)*exp(8.459+0.468*log(x[40])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[41]-1.04483+0.289)*exp(8.459+0.468*log(x[41])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[42]-1.04483+0.289)*exp(8.459+0.468*log(x[42])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[43]-1.04483+0.289)*exp(8.459+0.468*log(x[43])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[44]-1.04483+0.289)*exp(8.459+0.468*log(x[44])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[45]-1.04483+0.289)*exp(8.459+0.468*log(x[45])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[46]-1.04483+0.289)*exp(8.459+0.468*log(x[46])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[47]-1.04483+0.289)*exp(8.459+0.468*log(x[47])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[48]-1.04483+0.289)*exp(8.459+0.468*log(x[48])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[49]-1.04483+0.289)*exp(8.459+0.468*log(x[49])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[50]-1.04483+0.289)*exp(8.459+0.468*log(x[50])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[51]-1.04483+0.289)*exp(8.459+0.468*log(x[51])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[52]-1.04483+0.289)*exp(8.459+0.468*log(x[52])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[53]-1.04483+0.289)*exp(8.459+0.468*log(x[53])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[54]-1.04483+0.289)*exp(8.459+0.468*log(x[54])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[55]-1.04483+0.289)*exp(8.459+0.468*log(x[55])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[56]-1.04483+0.289)*exp(8.459+0.468*log(x[56])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[57]-1.04483+0.289)*exp(8.459+0.468*log(x[57])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[58]-1.04483+0.289)*exp(8.459+0.468*log(x[58])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[59]-1.04483+0.289)*exp(8.459+0.468*log(x[59])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[60]-1.04483+0.289)*exp(8.459+0.468*log(x[60])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[61]-1.04483+0.289)*exp(8.459+0.468*log(x[61])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[62]-1.04483+0.289)*exp(8.459+0.468*log(x[62])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[63]-1.04483+0.289)*exp(8.459+0.468*log(x[63])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[64]-1.04483+0.289)*exp(8.459+0.468*log(x[64])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[65]-1.04483+0.289)*exp(8.459+0.468*log(x[65])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[66]-1.04483+0.289)*exp(8.459+0.468*log(x[66])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[67]-1.04483+0.289)*exp(8.459+0.468*log(x[67])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[68]-1.04483+0.289)*exp(8.459+0.468*log(x[68])+1.07743-0.83) +
plogis(0.872 + 0.0000471*x[69]-1.04483+1.081)*exp(8.459+0.468*log(x[69])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[70]-1.04483+1.081)*exp(8.459+0.468*log(x[70])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[71]-1.04483+1.081)*exp(8.459+0.468*log(x[71])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[72]-1.04483+1.081)*exp(8.459+0.468*log(x[72])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[73]-1.04483+1.081)*exp(8.459+0.468*log(x[73])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[74]-1.04483+1.081)*exp(8.459+0.468*log(x[74])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[75]-1.04483+1.081)*exp(8.459+0.468*log(x[75])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[76]-1.04483+1.081)*exp(8.459+0.468*log(x[76])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[77]-1.04483+1.081)*exp(8.459+0.468*log(x[77])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[78]-1.04483+1.081)*exp(8.459+0.468*log(x[78])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[79]-1.04483+1.081)*exp(8.459+0.468*log(x[79])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[80]-1.04483+1.081)*exp(8.459+0.468*log(x[80])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[81]-1.04483+1.081)*exp(8.459+0.468*log(x[81])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[82]-1.04483+1.081)*exp(8.459+0.468*log(x[82])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[83]-1.04483+1.081)*exp(8.459+0.468*log(x[83])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[84]-1.04483+1.081)*exp(8.459+0.468*log(x[84])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[85]-1.04483+1.081)*exp(8.459+0.468*log(x[85])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[86]-1.04483+1.081)*exp(8.459+0.468*log(x[86])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[87]-1.04483+1.081)*exp(8.459+0.468*log(x[87])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[88]-1.04483+1.081)*exp(8.459+0.468*log(x[88])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[89]-1.04483+1.081)*exp(8.459+0.468*log(x[89])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[90]-1.04483+1.081)*exp(8.459+0.468*log(x[90])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[91]-1.04483+1.081)*exp(8.459+0.468*log(x[91])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[92]-1.04483+1.081)*exp(8.459+0.468*log(x[92])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[93]-1.04483+1.081)*exp(8.459+0.468*log(x[93])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[94]-1.04483+1.081)*exp(8.459+0.468*log(x[94])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[95]-1.04483+1.081)*exp(8.459+0.468*log(x[95])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[96]-1.04483+1.081)*exp(8.459+0.468*log(x[96])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[97]-1.04483+1.081)*exp(8.459+0.468*log(x[97])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[98]-1.04483+1.081)*exp(8.459+0.468*log(x[98])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[99]-1.04483+1.081)*exp(8.459+0.468*log(x[99])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[100]-1.04483+1.081)*exp(8.459+0.468*log(x[100])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[101]-1.04483+1.081)*exp(8.459+0.468*log(x[101])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[102]-1.04483+1.081)*exp(8.459+0.468*log(x[102])+1.07743-0.637) +
plogis(0.872 + 0.0000471*x[103]-1.04483-1.978)*exp(8.459+0.468*log(x[103])+1.07743+0.336) +
plogis(0.872 + 0.0000471*x[104]-1.04483-1.978)*exp(8.459+0.468*log(x[104])+1.07743+0.336) +
plogis(0.872 + 0.0000471*x[105]-1.04483-1.978)*exp(8.459+0.468*log(x[105])+1.07743+0.336) +
plogis(0.872 + 0.0000471*x[106]-1.04483-1.978)*exp(8.459+0.468*log(x[106])+1.07743+0.336) +
plogis(0.872 + 0.0000471*x[107]-1.04483+0.289-1.978)*exp(8.459+0.468*log(x[107])+1.07743-0.83+0.336) +
plogis(0.872 + 0.0000471*x[108]-1.04483+0.289-1.978)*exp(8.459+0.468*log(x[108])+1.07743-0.83+0.336) +
plogis(0.872 + 0.0000471*x[109]-1.04483+0.289-1.978)*exp(8.459+0.468*log(x[109])+1.07743-0.83+0.336) +
plogis(0.872 + 0.0000471*x[110]-1.04483+0.289-1.978)*exp(8.459+0.468*log(x[110])+1.07743-0.83+0.336) +
plogis(0.872 + 0.0000471*x[111]-1.04483+1.081-1.978)*exp(8.459+0.468*log(x[111])+1.07743-0.637+0.336) +
plogis(0.872 + 0.0000471*x[112]-1.04483+1.081-1.978)*exp(8.459+0.468*log(x[112])+1.07743-0.637+0.336) +
plogis(0.872 + 0.0000471*x[113]-1.04483+1.081-1.978)*exp(8.459+0.468*log(x[113])+1.07743-0.637+0.336) +
plogis(0.872 + 0.0000471*x[114]-1.04483+1.081-1.978)*exp(8.459+0.468*log(x[114])+1.07743-0.637+0.336)) )
}
# constraint function
eval_g0 <- function(x)
{
return(x[1]+x[2]+x[3]+x[4]+x[5]+x[6]+x[7]+x[8]+x[9]+x[10]+x[11]+x[12]+x[13]+x[14]+x[15]+x[16]+x[17]+x[18]+x[19]+x[20]+x[21]+x[22]+x[23]+x[24]+x[25]+x[26]+x[27]+x[28]+x[29]+x[30]+x[31]+x[32]+x[33]+x[34]+x[35]+x[36]+x[37]+x[38]+x[39]+x[40]+x[41]+x[42]+x[43]+x[44]+x[45]+x[46]+x[47]+x[48]+x[49]+x[50]+x[51]+x[52]+x[53]+x[54]+x[55]+x[56]+x[57]+x[58]+x[59]+x[60]+x[61]+x[62]+x[63]+x[64]+x[65]+x[66]+x[67]+x[68]+x[69]+x[70]+x[71]+x[72]+x[73]+x[74]+x[75]+x[76]+x[77]+x[78]+x[79]+x[80]+x[81]+x[82]+x[83]+x[84]+x[85]+x[86]+x[87]+x[88]+x[89]+x[90]+x[91]+x[92]+x[93]+x[94]+x[95]+x[96]+x[97]+x[98]+x[99]+x[100]+x[101]+x[102]+x[103]+x[104]+x[105]+x[106]+x[107]+x[108]+x[109]+x[110]+x[111]+x[112]+x[113]+x[114]-80000)
}
# Set optimization options.
opts <- list( "algorithm"= "NLOPT_GN_ISRES",
"xtol_rel"= 1.0e-7,
"maxeval"= 760000,
"print_level" = 0 )
# Solve
res1 <- nloptr( x0=c(500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500,500),
eval_f=eval_f0,
lb = c(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0),
ub = c(30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000,30000),
eval_g_ineq = eval_g0,
opts = opts)
print(res1)
The results are below:
Minimization using NLopt version 2.4.2
NLopt solver status: 5 ( NLOPT_MAXEVAL_REACHED: Optimization stopped because maxeval (above) was
reached. )
Number of Iterations....: 760000
Termination conditions: xtol_rel: 0.000000000000001 maxeval: 760000
Number of inequality constraints: 1
Number of equality constraints: 0
Current value of objective function: -4536536.34788097
Current value of controls: 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100
100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100
100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100
100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100
100 100 100 100 100 100 100 100 100 100 100 100 100 100
Could anyone take a look at my code and let me know whether I am doing anything wrong? Do I need to specify more iterations?
Edit: As #AirSquid suggested, I tried cutting the number of options down to 6 instead and that seems to have worked. So I need to understand why it doesn't work for a larger number of options
This is an extremely easy linear constrained problem. A solver like IPOPT solves this in no time: 0.004 seconds, 8 iterations (I used your exact formulation, with a lower bound on the variables of 0.0001 to protect the log -- I hope I transcribed the problem correctly). Note that nloptr contains largely somewhat simple algorithms. So I suggest picking a better solver.
This is Ipopt version 3.14.6, running with linear solver ma27.
Number of nonzeros in equality constraint Jacobian...: 0
Number of nonzeros in inequality constraint Jacobian.: 114
Number of nonzeros in Lagrangian Hessian.............: 114
Total number of variables............................: 114
variables with only lower bounds: 0
variables with lower and upper bounds: 114
variables with only upper bounds: 0
Total number of equality constraints.................: 0
Total number of inequality constraints...............: 1
inequality constraints with only lower bounds: 0
inequality constraints with lower and upper bounds: 0
inequality constraints with only upper bounds: 1
iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls
0 -1.6299318e+07 0.00e+00 5.96e+01 0.0 0.00e+00 - 0.00e+00 0.00e+00 0
1 -1.9108212e+07 0.00e+00 5.08e+01 3.0 9.26e+04 - 1.00e+00 2.46e-01f 1
2 -1.9489289e+07 0.00e+00 4.44e+01 2.0 1.58e+03 - 1.00e+00 1.44e-01f 1
3 -2.0776301e+07 0.00e+00 2.03e+01 1.4 9.55e+02 - 1.00e+00 1.00e+00f 1
4 -2.0890126e+07 0.00e+00 6.06e+00 -0.1 9.27e+02 - 1.00e+00 8.42e-01f 1
5 -2.0908516e+07 0.00e+00 5.78e-01 -1.4 5.14e+02 - 9.89e-01 1.00e+00f 1
6 -2.0908784e+07 0.00e+00 8.26e-03 -3.0 7.22e+01 - 9.97e-01 1.00e+00f 1
7 -2.0908784e+07 0.00e+00 6.72e-06 -4.9 1.05e+00 - 9.96e-01 1.00e+00f 1
8 -2.0908784e+07 0.00e+00 1.54e-10 -10.7 2.17e-04 - 1.00e+00 1.00e+00f 1
Number of Iterations....: 8
(scaled) (unscaled)
Objective...............: -6.4238210381431347e+06 -2.0908784212199997e+07
Dual infeasibility......: 1.5371171002698247e-10 5.0031359167058148e-10
Constraint violation....: 0.0000000000000000e+00 0.0000000000000000e+00
Variable bound violation: 0.0000000000000000e+00 0.0000000000000000e+00
Complementarity.........: 5.5985347935841295e-10 1.8222574260471783e-09
Overall NLP error.......: 5.5985347935841295e-10 1.8222574260471783e-09
Number of objective function evaluations = 9
Number of objective gradient evaluations = 9
Number of equality constraint evaluations = 0
Number of inequality constraint evaluations = 9
Number of equality constraint Jacobian evaluations = 0
Number of inequality constraint Jacobian evaluations = 9
Number of Lagrangian Hessian evaluations = 8
Total seconds in IPOPT = 0.004
EXIT: Optimal Solution Found.
I am trying to order by date [SLED/BB] some records and to calculate and select the ones in the 75th percentile of [Blocked Quantity].
On ACCESS SQL I am not able to select in the WHERE clause the Percentile > 0.75,
I have read that this is not possible on this sw but how can I replicate this result?
p.s. I would like to avoid too many Subqueries given that this query is already quite heavy for my dataset.
SELECT
a1.[SLED/BBD],
SUM([Blocked Quantity]) AS [Blocked Qty],
(SELECT sum([Blocked Quantity])
FROM [Query002-Batches Historical] AS a2
WHERE a1.[SLED/BBD] <= a2.[SLED/BBD]) AS RunningTotal,
(SELECT SUM([Blocked Quantity])
FROM [Query002-Batches Historical]
GROUP BY 1) AS Total,
FORMAT(RunningTotal/Total,"Percent") AS Percentile
FROM [Query002-Batches Historical] AS a1
WHERE a1.[Blocked Quantity]>0 AND Percentile > 0.75
GROUP BY a1.[SLED/BBD]
ORDER BY a1.[SLED/BBD];
Though not providing the result directly, this function may help you:
Public Function GetQuartile( _
ByVal strTable As String, _
ByVal strField As String, _
ByVal bytQuartile As Byte, _
Optional ByVal bytMethod As Byte, _
Optional ByVal strFilter As String) _
As Double
' strTable : Name of the table/query to analyze.
' strField : Name of the field to analyze.
' bytQuartile: Which min/max or median/quartile to calculate.
' bytMethod: Method for calculation of lower/higher quartile.
' strFilter: Optional filter expression.
'
' Returns:
' Minimum, maximum, median or upper/lower quartile
' of strField of strTable filtered on strFilter.
'
' 2006-03-05. Cactus Data ApS, CPH.
' Reference for methods for calculation as explained here:
' http://www.daheiser.info/excel/notes/noteh.pdf
' Note: Table H-4, p. 4, has correct data for dataset 1-96 while
' datasets 1-100 to 1-97 actually are datasets 1-99 to 1-96
' shifted one column left.
' Thus, the dataset 1-100 is missing.
'
' Method 3b is not implemented as no one seems to use it.
' Neither are no example data given.
'
' Further notes on methods here:
' http://mathforum.org/library/drmath/view/60969.html
' http://www.haiweb.org/medicineprices/manual/quartiles_iTSS.pdf
'
' Data must be in ascending order by strField.
' L: Q1, Lower quartile.
' H: Q3, Higher quartile.
' M: Q2, Median.
' n: Count of elements.
' p: Calculated position of quartile.
' j: Element of dataset.
' g: Decimal part of p
' to be used for interpolation between j and j+1.
' Basic operation.
' Constant values mimic those of Excel's Quartile() function.
' Find median.
Const cbytQuartMedian As Byte = 2
' Find lower (first) quartile.
Const cbytQuartLow As Byte = 1
' Find upper (third) quartile.
Const cbytQuartHigh As Byte = 3
' Find minimum value.
Const cbytQuartMinimum As Byte = 0
' Find maximum value.
Const cbytQuartMaximum As Byte = 4
' Define default operation.
Const cbytQuartDefault = cbytQuartMedian
' Quartile calculation methods.
' Step. Mendenhall and Sincich method.
' SAS #3.
' Round up to actual element of dataset.
' L: -Int(-n/4)
' H: n-Int(-n/4)
Const cbytMethodMendenhallSincich As Byte = 1
' Average step.
' SAS #5, Minitab (%DESCRIBE), GLIM (percentile).
' Add bias of one or two on basis of n/4.
' L: (Int((n+1)/4)+Int(n/4))/2+1
' H: n-(Int((n+1)/4)+Int(n/4))/2+1
Const cbytMethodAverage As Byte = 2
' Nearest integer to np.
' SAS #2.
' Round to nearest integer on basis of n/4.
' L: Int((n+2)/4)
' H: n-Int((n+2)/4)
' Note:
' Reference contains an error in example data.
' Dataset 1-100 to 1-97 (is really 1-99 to 1-96!) should read:
' 25 25 24 24
Const cbytMethodNearestInteger As Byte = 3
' Parzen method.
' Method 1 with interpolation.
' SAS #1.
' L: n/4
' H: 3n/4
Const cbytMethodParzen As Byte = 4
' Hazen method.
' Values midway between method 1 steps.
' GLIM (interpolate).
' Add bias of 2, don't round to actual element of dataset.
' L: (n+2)/4
' H: 3(n+2)/4
Const cbytMethodHazen As Byte = 5
' Weibull method.
' SAS #4. Minitab (DECRIBE), SPSS, BMDP.
' Add bias of 1, don't round to actual element of dataset.
' L: (n+1)/4
' H: 3(n+1)/4
Const cbytMethodWeibull As Byte = 6
' Freund, J. and Perles, B., Gumbell method.
' S-PLUS, R, Excel, Star Office Calc.
' Add bias of 3, don't round to actual element of dataset.
' L: (n+3)/4
' H: (3n+1)/4
Const cbytMethodFreundPerles As Byte = 7
' Median Position.
' Median unbiased.
' L: (3n+5)/12
' H: (9n+7)/12
Const cbytMethodMedianPosition As Byte = 8
' Bernard and Bos-Levenbach.
' L: (n/4)+0.4
' H: (3n/4)/+0.6
' Note:
' Reference claims L to be (n/4)+0.31.
Const cbytMethodBernardLevenbach As Byte = 9
' Blom's Plotting Position.
' Better approximation when the distribution is normal.
' L: (4n+7)/16
' H: (12n+9)/16
Const cbytMethodBlom As Byte = 10
' Moore's first method.
' Add bias of one half step.
' L: (n+0.5)/4
' H: n-(n+0.5)/4
Const cbytMethodMoore1 As Byte = 11
' Moore's second method.
' Add bias of one or two steps on basis of (n+1)/4.
' L: (Int((n+1)/4)+Int(n/4))/2+1
' H: n-(Int((n+1)/4)+Int(n/4))/2+1
Const cbytMethodMoore2 As Byte = 12
' John Tukey's method.
' Include median from odd dataset in dataset for quartile.
' L: (1-Int(-n/2))/2
' H: n-(1-Int(-n/2))/2
Const cbytMethodTukey As Byte = 13
' Moore and McCabe (M & M), variation of John Tukey's method.
' TI-83.
' Exclude median from odd dataset in dataset for quartile.
' L: (Int(n/2)+1)/2
' H: n-(Int(n/2)+1)/2
Const cbytMethodTukeyMM As Byte = 14
' Additional variations between Weibull's and Hazen's methods, from
' (i-0.000)/(n+1.00)
' to
' (i-0.500)/(n+0.00)
'
' Variation of Weibull.
' L: n(n/4-0)/(n+1)
' H: n(3n/4-0)/(n+1)
Const cbytMethodModWeibull As Byte = 15
' Variation of Blom.
' L: n(n/4-3/8)/(n+1/4)
' H: n(3n/4-3/8)/(n+1/4)
Const cbytMethodModBlom As Byte = 16
' Variation of Tukey.
' L: n(n/4-1/3)/(n+1/3)
' H: n(3n/4-1/3)/(n+1/3)
Const cbytMethodModTukey As Byte = 17
' Variation of Cunnane.
' L: n(n/4-2/5)/(n+1/5)
' H: n(3n/4-2/5)/(n+1/5)
Const cbytMethodModCunnane As Byte = 18
' Variation of Gringorten.
' L: n(n/4-0.44)/(n+0.12)
' H: n(3n/4-0.44)/(n+0.12)
Const cbytMethodModGringorten As Byte = 19
' Variation of Hazen.
' L: n(n/4-1/2)/n
' H: n(3n/4-1/2)/n
Const cbytMethodModHazen As Byte = 20
' Define default method to calculate quartiles.
Const cbytMethodDefault = cbytMethodFreundPerles
Static dbs As DAO.Database
Static rst As DAO.Recordset
Dim strSQL As String
Dim lngNumber As Long
Dim dblPosition As Double
Dim lngPosition As Long
Dim dblInterpol As Double
Dim dblValueOne As Double
Dim dblValueTwo As Double
Dim dblQuartile As Double
' Use default calculation if choice of calculation is outside range.
If bytQuartile > 4 Then
bytQuartile = cbytQuartDefault
End If
' Use default method if choice of method is outside range.
If bytMethod = 0 Or bytMethod > 20 Then
bytMethod = cbytMethodDefault
End If
If dbs Is Nothing Then
Set dbs = CurrentDb()
End If
If Len(strTable) > 0 And Len(strField) > 0 Then
strSQL = "SELECT [" & strField & "] FROM [" & strTable & "] "
strSQL = strSQL & "WHERE ([" & strField & "] Is Not Null) "
If Len(strFilter) > 0 Then
strSQL = strSQL & "AND (" & strFilter & ") "
End If
strSQL = strSQL & "ORDER BY [" & strField & "];"
Set rst = dbs.OpenRecordset(strSQL)
With rst
If Not .EOF = True Then
If bytQuartile = cbytQuartMinimum Then
' No need to count records.
lngNumber = 1
Else
' Count records.
.MoveLast
lngNumber = .RecordCount
End If
Select Case bytQuartile
Case cbytQuartMinimum
' Current record is first record.
' Read value of this record.
Case cbytQuartMaximum
' Current record is last record.
' Read value of this record.
Case cbytQuartMedian
' Locate position of median.
dblPosition = (lngNumber + 1) / 2
Case cbytQuartLow
Select Case bytMethod
Case cbytMethodMendenhallSincich
dblPosition = -Int(-lngNumber / 4)
Case cbytMethodAverage
dblPosition = (Int((lngNumber + 1) / 4) + Int(lngNumber / 4)) / 2 + 1
Case cbytMethodNearestInteger
dblPosition = Int((lngNumber + 2) / 4)
Case cbytMethodParzen
dblPosition = lngNumber / 4
Case cbytMethodHazen
dblPosition = (lngNumber + 2) / 4
Case cbytMethodWeibull
dblPosition = (lngNumber + 1) / 4
Case cbytMethodFreundPerles
dblPosition = (lngNumber + 3) / 4
Case cbytMethodMedianPosition
dblPosition = (3 * lngNumber + 5) / 12
Case cbytMethodBernardLevenbach
dblPosition = (lngNumber / 4) + 0.4
Case cbytMethodBlom
dblPosition = (4 * lngNumber + 7) / 16
Case cbytMethodMoore1
dblPosition = (lngNumber + 0.5) / 4
Case cbytMethodMoore2
dblPosition = (Int((lngNumber + 1) / 4) + Int(lngNumber / 4)) / 2 + 1
Case cbytMethodTukey
dblPosition = (1 - Int(-lngNumber / 2)) / 2
Case cbytMethodTukeyMM
dblPosition = (Int(lngNumber / 2) + 1) / 2
Case cbytMethodModWeibull
dblPosition = lngNumber * (lngNumber / 4) / (lngNumber + 1)
Case cbytMethodModBlom
dblPosition = lngNumber * (lngNumber / 4 - 3 / 8) / (lngNumber + 1 / 4)
Case cbytMethodModTukey
dblPosition = lngNumber * (lngNumber / 4 - 1 / 3) / (lngNumber + 1 / 3)
Case cbytMethodModCunnane
dblPosition = lngNumber * (lngNumber / 4 - 2 / 5) / (lngNumber + 1 / 5)
Case cbytMethodModGringorten
dblPosition = lngNumber * (lngNumber / 4 - 0.44) / (lngNumber + 0.12)
Case cbytMethodModHazen
dblPosition = lngNumber * (lngNumber / 4 - 1 / 2) / lngNumber
End Select
Case cbytQuartHigh
Select Case bytMethod
Case cbytMethodMendenhallSincich
dblPosition = lngNumber - (-Int(-lngNumber / 4))
Case cbytMethodAverage
dblPosition = lngNumber - (Int((lngNumber + 1) / 4) + Int(lngNumber / 4)) / 2 + 1
Case cbytMethodNearestInteger
dblPosition = lngNumber - Int((lngNumber + 2) / 4)
Case cbytMethodParzen
dblPosition = 3 * lngNumber / 4
Case cbytMethodHazen
dblPosition = 3 * (lngNumber + 2) / 4
Case cbytMethodWeibull
dblPosition = 3 * (lngNumber + 1) / 4
Case cbytMethodFreundPerles
dblPosition = (3 * lngNumber + 1) / 4
Case cbytMethodMedianPosition
dblPosition = (9 * lngNumber + 7) / 12
Case cbytMethodBernardLevenbach
dblPosition = (3 * lngNumber / 4) + 0.6
Case cbytMethodBlom
dblPosition = (12 * lngNumber + 9) / 16
Case cbytMethodMoore1
dblPosition = lngNumber - (lngNumber + 0.5) / 4
Case cbytMethodMoore2
dblPosition = lngNumber - (Int((lngNumber + 1) / 4) + Int(lngNumber / 4)) / 2 + 1
Case cbytMethodTukey
dblPosition = lngNumber - (1 - Int(-lngNumber / 2)) / 2
Case cbytMethodTukeyMM
dblPosition = lngNumber - (Int(lngNumber / 2) + 1) / 2
Case cbytMethodModWeibull
dblPosition = lngNumber * (3 * lngNumber / 4) / (lngNumber + 1)
Case cbytMethodModBlom
dblPosition = lngNumber * (3 * lngNumber / 4 - 3 / 8) / (lngNumber + 1 / 4)
Case cbytMethodModTukey
dblPosition = lngNumber * (3 * lngNumber / 4 - 1 / 3) / (lngNumber + 1 / 3)
Case cbytMethodModCunnane
dblPosition = lngNumber * (3 * lngNumber / 4 - 2 / 5) / (lngNumber + 1 / 5)
Case cbytMethodModGringorten
dblPosition = lngNumber * (3 * lngNumber / 4 - 0.44) / (lngNumber + 0.12)
Case cbytMethodModHazen
dblPosition = lngNumber * (3 * lngNumber / 4 - 1 / 2) / lngNumber
End Select
End Select
Select Case bytQuartile
Case cbytQuartMinimum, cbytQuartMaximum
' Read current row.
Case Else
.MoveFirst
' Find position of first observation to retrieve.
' If lngPosition is 0, then upper position is first record.
' If lngPosition is not 0 and position is not an integer, then
' read the next observation too.
lngPosition = Fix(dblPosition)
dblInterpol = dblPosition - lngPosition
If lngNumber = 1 Then
' Nowhere else to move.
If dblInterpol < 0 Then
' Prevent values to be created by extrapolation beyond zero from observation one
' for these methods:
' cbytMethodModBlom
' cbytMethodModTukey
' cbytMethodModCunnane
' cbytMethodModGringorten
' cbytMethodModHazen
'
' Comment this line out, if reading by extrapolation *is* requested.
dblInterpol = 0
End If
ElseIf lngPosition > 1 Then
' Move to record to read.
.Move lngPosition - 1
End If
End Select
' Retrieve value from first observation.
dblValueOne = .Fields(0).Value
Select Case bytQuartile
Case cbytQuartMinimum, cbytQuartMaximum
dblQuartile = dblValueOne
Case Else
If dblInterpol = 0 Then
' Only one observation to read.
If lngPosition = 0 Then
' Return 0.
Else
dblQuartile = dblValueOne
End If
Else
If lngPosition = 0 Then
' No first observation to retrieve.
dblValueTwo = dblValueOne
If dblValueOne > 0 Then
' Use 0 as other observation.
dblValueOne = 0
Else
dblValueOne = 2 * dblValueOne
End If
Else
' Move to next observation.
.MoveNext
' Retrieve value from second observation.
dblValueTwo = .Fields(0).Value
End If
' For positive values interpolate between 0 and dblValueOne.
' For negative values interpolate between 2 * dblValueOne and dblValueOne.
' Calculate quartile using linear interpolation.
dblQuartile = dblValueOne + dblInterpol * CDec(dblValueTwo - dblValueOne)
End If
End Select
End If
.Close
End With
Else
' Reset.
Set rst = Nothing
Set dbs = Nothing
End If
''Set rst = Nothing
GetQuartile = dblQuartile
End Function
Unfortunately, the pdf referenced is off-line now and I've lost the copy I had.