db.runCommand({"group" :{
"ns" : "keyword_logs",
"key" : "keyword",
"initial" : {"keyword" : {}},
"$reduce" : function (doc , prev) {
if(doc.keyword != null && prev.keyword != null &&(doc.keyword in prev.keyword)) {
prev.keyword[doc.keyword]++;
}
else {
prev.keyword[doc.keyword] = 1 ;
}
},
"condition" : {"domain" : 40}
}})
this is errorMsg
{
"errmsg" : "exception: JavaScript property (name) contains a null char which is not allowed in BSON. {}",
"code" : 16985,
"ok" : 0
}
Thanks Gibert Sun , in his help ,we find the error:
This is the code to find the error data
db.runCommand({
"group" :{
"ns" : "keyword_logs",
"key" : "keyword",
"initial" : {"error" : []},
"$reduce" : function (doc , prev) {
if (/\u0000/.test(doc.keyword)) prev.error.push(doc);
},
"condition" : {"domain" : 10}
}
})
this is the new code for the search
db.runCommand({
"group" :{
"ns" : "keyword_logs",
"key" : "keyword",
"initial" : {"keyword" : {}},
"$reduce" : function (doc , prev) {
if (/\u0000/.test(doc.keyword)) return;
var kw = doc.keyword;
if(kw in prev.keyword){
prev.keyword[doc.keyword]++;
}else{
prev.keyword[doc.keyword] = 1 ;
}
},
"condition" : {"domain" : 10}
}
})
Related
I am trying to use mongoosastic search() but it doesn't give me the result.
Data is saved in MongoDB as well as elasticsearch.
Below is the mongoose schema
const mongoose = require('mongoose');
const mongoosastic = require('mongoosastic');
var Schema = mongoose.Schema;
mongoose.connect(`mongodb://localhost:27017/elasticsearch`, function(err) {
if (err) {
console.error(err);
}
console.log('connected.... unless you see an error the line before this!');
});
var ItemSchema = new Schema({
name : {type : String, es_indexed : true},
description : {type : String, es_indexed : true}
});
ItemSchema.plugin(mongoosastic, {
"host": "localhost",
"port": 9200
});
var ItemModel = mongoose.model("Item", ItemSchema);
ItemModel.createMapping((err, mapping) => {
console.log('mapping created');
});
Search Query that I'm trying to use to get the desired search results.
app.get("/search", (req, res) => {
console.log(req.query.item);
ItemModel.search({
"multi_match": {
"fields": ["multi_field"],
"query": req.query.item,
"fuzziness": "AUTO"
}
},
function (err, results) {
if (err) {
return console.log(JSON.stringify(err, null, 4));
}
return console.log(JSON.stringify(results, null, 4));
});
});
Below are the results that are stored on mongoDB.
{ "_id" : ObjectId("627204f3ec9b3ac2cee45c75"), "name" : "water bottle", "description" : "used to store water", "__v" : 0 }
{ "_id" : ObjectId("62720516ec9b3ac2cee45c77"), "name" : "chair", "description" : "used to sit", "__v" : 0 }
{ "_id" : ObjectId("6272060aec9b3ac2cee45c84"), "name" : "Dettol hand sanitizer", "description" : "used to sanitize hands", "__v" : 0 }
{ "_id" : ObjectId("62724a5b9c2c700433afab84"), "name" : "laptop charger", "description" : "used to charge laptop", "__v" : 0 }
Mapping in Elasticsearch
{
"items" : {
"mappings" : {
"properties" : {
"description" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
}
}
}
Data set saved on elasticsearch
Data saved on Elasticsearch
Finally this Worked for me...
const item = req.params.item;
const results = await ItemModel.search({
query_string: {
query: item
}
});
if (results.body.hits.hits) {
res.json(results.body.hits.hits);
} else {
res.status(400).json({ success: false });
}
I'v tried to perform the following query :
db.getCollection('fxh').find({"username": "user1", "pf.acc.accnbr" : 915177},{userid: true, "pf.pfid": true, "pf.acc.accid":true})
and my collection is the following :
{
"_id" : ObjectId("5932fd8f381d4c0a7de21942"),
"userid" : 1496513894,
"username" : "user1",
"email" : "user1#gmail.com",
"fullname" : "User 1",
"pf" : {
"acc" : [
{
"cyc" : [
{
"det" : {
"status" : "New",
"dcycid" : 1496513941
},
"status" : "New",
"name" : "QPT202017_M1",
"cycid" : 1496513940
}
],
"status" : "New",
"accnbr" : 915177,
"accid" : 1496513939
},
{
"cyc" : [
{
"det" : {
"status" : "New",
"dcycid" : 1496552643
},
"status" : "New",
"name" : "QPT202017_S8",
"cycid" : 1496552642
}
],
"status" : "New",
"accnbr" : 73497,
"accid" : 1496552641
}
],
"pfid" : 1496513935,
},
"lastupdate" : ISODate("2017-06-03T18:18:55.080Z"),
"__v" : 0
}
When I execute the query the result is the following :
{
"_id" : ObjectId("5932fd8f381d4c0a7de21942"),
"userid" : 1496513894,
"portfolio" : {
"acc" : [
{
"accid" : 1496513939
},
{
"accid" : 1496552641
}
],
"pfid" : 1496513935
}
}
And my problem is that I need to see only the concerned accid and the result returns the all accid !.
Any idea how just to return the selected accid of accnbr ?
NB : I have also tried to add $ sign at the end of my query , it
selects the right acc but it returns the all objects or I need just
only ONE returned object.
On 6/5/17
I also used the aggregate command instead of find and it get result by using this :
db.getCollection('fxh').aggregate([ { $unwind : "$pf.acc"} , { $match : {"username":"adh1", "pf.acc.accbr": 915177 } }, {$project : {_id:0, accid: "$pf.acc.accid"}}])
But could NOT get a lower level result, when I ran this :
db.getCollection('fxh').aggregate([ { $unwind : "$pf.acc.cyc"} , { $match : {"username":"adh1", "pf.acc.accbr": 915177, "pf.acc.cyc.name": "QPT202017_M1" } }, {$project : {_id:0, cycid: "$pf.acc.cyc.cycid"}}])
Any idea ?
You can try the below aggregation pipeline.
The idea is to $unwind one nested level at a time, starting from the outermost to the innermost.
For each nested level unwinding, you can apply the$match to limit the documents and continue till you have the desired shape.
You can $group it together at the end to get back to the original shape.
db.getCollection('fxh').aggregate([
{ $match : {"username":"adh1"} },
{ $unwind : "$pf.acc"} ,
{ $match : {"pf.acc.accbr": 915177 } },
{ $unwind : "$pf.acc.cyc"},
{ $match : {"pf.acc.cyc.name": "QPT202017_M1" } },
{$project : {_id:0, accid: "$pf.acc.accid", cycid: "$pf.acc.cyc.cycid"}}])
I have collection with following data set
MongoDB Enterprise > db.UCODEPatter.find()
{ "_id" : { "UCODE" : NumberLong("233220524641368") }, "value" : 117 }
{ "_id" : { "UCODE" : NumberLong("233222325602424") }, "value" : 71 }
{ "_id" : { "UCODE" : NumberLong("233222325601932") }, "value" : 69 }
{ "_id" : { "UCODE" : NumberLong("233222325601947") }, "value" : 59 }
{ "_id" : { "UCODE" : NumberLong("233222325602414") }, "value" : 58 }
{ "_id" : { "UCODE" : NumberLong("233222325602419") }, "value" : 49 }
{ "_id" : { "UCODE" : NumberLong("233222325602418") }, "value" : 48 }
{ "_id" : { "UCODE" : NumberLong("233220535710165") }, "value" : 47 }
{ "_id" : { "UCODE" : NumberLong("233222325602405") }, "value" : 46 }
{ "_id" : { "UCODE" : NumberLong("233222325602420") }, "value" : 45 }
{ "_id" : { "UCODE" : NumberLong("233220537026120") }, "value" : 45 }
{ "_id" : { "UCODE" : NumberLong("233222686003297") }, "value" : 44 }
{ "_id" : { "UCODE" : NumberLong("233222325602417") }, "value" : 44 }
i am try to export above data as CSV file using PHP , here my php code
<?php
header('Content-Type: application/csv');
header('Content-Disposition: attachment; filename=example.csv');
header('Pragma: no-cache');
$database = "LogData";
$colName = "UCODEPatter";
$connection = new MongoClient();
$collection = $connection->$colName->$database;
$cursor = $collection->find();
foreach($cursor as $cur)
echo '"'.$cur["UCODE"].'","'.$cur['value']."\"\n";
?>
when i used above code , i created csv file without data, please kindly help me to sort this
I think, you messed up with methods here:
$collection = $connection->$colName->$database;
There should be this:
$collection = $connection->selectDB($database)->selectCollection($colName);
And you didn't show that your collection exists in db named "LogData", so you should check this.
Select COUNT(distinct name)
From index1
Where date between X and y
And name in (Select name
From index1
Where date between p and s)
Equivalent query in elasticsearch ?
The Filter Aggregation may be the answer.
Something like this:
{
"size" : 0,
"query" : {
"filtered" : {
"query" : {
"match_all" : { }
},
"filter" : {
"range" : {
"date" : {
"from" : "2015-03-10T21:51:47.703-04:00",
"to" : "2015-03-20T21:51:47.727-04:00",
"include_lower" : true,
"include_upper" : true
}
}
}
}
},
"aggregations" : {
"names1" : {
"filter" : {
"range" : {
"date" : {
"from" : "2015-02-28T21:51:47.733-05:00",
"to" : "2015-03-20T21:51:47.734-04:00",
"include_lower" : true,
"include_upper" : true
}
}
},
"aggregations" : {
"names2" : {
"terms" : {
"field" : "name"
}
}
}
}
}
}
I am trying to create a aggregate results in elastic search but filter option is not working for me.
I can aggregate data without filter e.g.
select name , material ,sum(price)
from products group by name , material
curl -XGET 'http://localhost:9200/products/_search?pretty=true' -d'
{
"aggs" : {
"product" : {
"terms" : {
"field" : "name"
},
"aggs" : {
"material" : {
"terms" : {
"field" : "material"
},
"aggs" : {
"sum_price" : {
"sum" : {
"field" : "price"
}
}
}
}
}
}
},
"size" : 0
}'
but I am facing problems to write equivalent DSL query of :
select name , material ,sum(price)
from products
where material = "wood"
group by name , material
Should be something like this:
{
"query": {
"filtered": {
"query": {
"match_all": {}
},
"filter": {
"term": {
"material": "wood"
}
}
}
},
"aggs" : {
"product" : {
"terms" : {
"field" : "name"
},
"aggs" : {
"material" : {
"terms" : {
"field" : "material"
},
"aggs" : {
"sum_price" : {
"sum" : {
"field" : "price"
}
}
}
}
}
}
},
"size" : 0
}
Use a filter if you know the exact value and do not need a match, else use a match query instead of the filtered query.
You can use match
{
"query": {
"bool": {
"must": [
{
"match": {
"material": "wood"
}
}
],
"filter": [
{
"match_all": {}
},
]
}
},
"aggs" : {
"product" : {
"terms" : {
"field" : "name"
},
"aggs" : {
"material" : {
"terms" : {
"field" : "material"
},
"aggs" : {
"sum_price" : {
"sum" : {
"field" : "price"
}
}
}
}
}
}
},
"size" : 0
}