MongoDB multiple Lookup into same collection - sql

I have two collections Bill and Employee. Bill contains the information about the monthly student bill and Employee contains all types of people working in the school (Accountant, Teachers, Maintenance etc).
Bill has billVerifyBy and classteacher field which points to the records of Employees.
Bill collection
{
"_id": ObjectId("ab12dns..."), //mongoid
"studentname": "demoUser",
"class": { "section": "A"},
"billVerifiedBy": "121212",
"classteacher": "134239",
}
Employee collection
{
"_id": ObjectId("121212"), // random number
"name": "Darn Morphy",
"department": "Accounts",
"email": "dantest#test.com",
}
{
"_id": ObjectId("134239"),
"name": "Derreck",
"department": "Faculty",
"email": "derrect145#test.com",
}
I need to retrieve the Accounts and Teacher information related to a particular bill. I am using Mongodb lookup to get the information. However, I have to lookup to the same table twice since billVerifiedBy and classteacher belong to the same Employee tables as given below.
db.bill.aggregate([
{
$lookup: {"from": "employee", "localField": "billVerifiedBy", "foreignField": "_id", "as": "accounts"}},
},
{
$lookup: {"from": "employee", "localField": "classteacher", "foreignField": "_id", "as": "faculty"}},
},
{
$project: {
"studentname": 1,
"class": 1,
"verifiedUser": "$accounts.name",
"verifiedByEmail":"$accounts.email",
"facultyName": "$faculty.name",
"facultyEmail": "$faculty.email"
}
}
]
I don't know if this is the good way of arranging the Accounts and Faculty information in the single Employee collection. And is it right thing to lookup twice with same collection. Or should I create separate Accounts and Faculty collection and lookup with it. Please suggest what would be the best approach in terms of performance.

In mongodb, when you want to join multiple documents from the same collection, you can use "$lookup" with its "pipeline" and "let" options. It filters documents that you want to take with defined variables.
db.getCollection('Bill').aggregate([{
"$lookup": {
"as": "lookupUsers",
"from": "Employee",
// define variables that you need to use in pipeline to filter documents
"let": {
"verifier": "$billVerifiedBy",
"teacher": "$classteacher"
},
"pipeline": [{ // filter employees who you need to filter.
"$match": {
"$expr": {
"$or": [{
"$eq": ["$_id", "$$verifier"]
},
{
"$eq": ["$_id", "$$teacher"]
}
]
}
}
},
{ // combine filtered 2 documents in an employee array
"$group": {
"_id": "",
"employee": {
"$addToSet": {
"_id": "$_id",
"name": "$name",
"department": "$department",
"email": "$email"
}
}
}
},
{ // takes item from the array by predefined variable.
"$project": {
"_id": 0,
"billVerifiedBy": {
"$slice": [{
"$filter": {
"input": "$employee",
"cond": {
"$eq": ["$$this._id", "$$verifier"]
}
}
},
1
]
},
"classteacher": {
"$slice": [{
"$filter": {
"input": "$employee",
"cond": {
"$eq": ["$$this._id", "$$teacher"]
}
}
},
1
]
}
}
},
{
"$unwind": "$billVerifiedBy"
},
{
"$unwind": "$classteacher"
},
]
}
},
{
"$unwind": "$lookupUsers"
},
]);
Output is like that:
{
"_id": ObjectId("602916dcf4450742cdebe38d"),
"studentname": "demoUser",
"class": {
"section": "A"
},
"billVerifiedBy": ObjectId("6029172e9ea6c9d4776517ce"),
"classteacher": ObjectId("6029172e9ea6c9d4776517cf"),
"lookupUsers": {
"billVerifiedBy": {
"_id": ObjectId("6029172e9ea6c9d4776517ce"),
"name": "Darn Morphy",
"department": "Accounts",
"email": "dantest#test.com"
},
"classteacher": {
"_id": ObjectId("6029172e9ea6c9d4776517cf"),
"name": "Derreck",
"department": "Faculty",
"email": "derrect145#test.com"
}
}
}

Related

Get data from two collection with specific data Mongo

Get data from two collection from first collection(test1) all data and from second collection (test2) customer name using createdBy and updatedBy from test1 collection.
In createdBy and updatedBy I want fullname from test2 collection:
Test1 collection:
{
"_id": "kcXtyaB7jGPw9Ks",
"dateCreated": "2022-07-12T13:09:16.270Z",
"dateModified": "2022-07-12T13:09:16.270Z",
"data1": 1,
"data2": 100,
"data3": 5,
"createdBy": "xQQrzRgi8",
"updatedBy": "56sgAeKfx"
}
Test2 collection:
{
"_id": "xQQrzRgi8",
"fullName": "test name created"
},
{
"_id": "56sgAeKfx",
"fullName": "test name update"
}
Response be like:
{
"_id": "kcXtyaB7jGPw9Ks",
"dateCreated": "2022-07-12T13:09:16.270Z",
"dateModified": "2022-07-12T13:09:16.270Z",
"data1": 1,
"data2": 100,
"data3": 5,
"createdBy": "test name created",
"updatedBy": "test name update"
}
If I've understood correctly, you can use $lookup like this:
This query do a "join" between "Test1" and "Test2" using updatedBy and _id fields.
And after that get the first element in the result (I assume there were only one element because you are comparing with _id but if there is more than one you can use another way like $unwind) to output the value.
Edit: To get both values (created and updated) you can do a second $lookup.
Now the query:
Get the updatedBy name from field _id in Test2.
Set value into field updatedBy.
Get the createdBy name from field _id in Test2.
Set value into field createdBy.
Use $project to not output result.
db.Test1.aggregate([
{
"$lookup": {
"from": "Test2",
"localField": "updatedBy",
"foreignField": "_id",
"as": "result"
}
},
{
"$set": {
"updatedBy": {
"$first": "$result.fullName"
}
}
},
{
"$lookup": {
"from": "Test2",
"localField": "createdBy",
"foreignField": "_id",
"as": "result"
}
},
{
"$set": {
"createdBy": {
"$first": "$result.fullName"
}
}
},
{
"$project": {
"result": 0
}
}
])
Example here
I solved my query with below mongo query:
db.Test1.aggregate([
{
$lookup: {
from: "Test2",
localField: "updatedBy",
foreignField: "_id",
as: "updatedByName",
},
},
{
$lookup: {
from: "Test2",
localField: "createdBy",
foreignField: "_id",
as: "createdByName",
},
},
{
$set: {
updatedBy: {
$first: "$updatedByName.fullName",
},
},
},
{
$set: {
createdBy: {
$first: "$createdByName.fullName",
},
},
},
{
$project: {
updatedByName: 0,
createdByName: 0,
},
}
])
Here is Solved query https://mongoplayground.net/p/7Ekh-q8tkTy

How to avoid the duplicated data entry after parsing json in kusto?

I have following sample json data.
{
"data": {
"type": "ABC",
"id": "17495500314",
"attributes": {
[!["event": "update",
"gps_vali][1]][1]d": true,
"gps": {
"distance_diff": 6.48,
"total_distance": 848.6
},
"hdop": 79,
"fuel_level": 46.8,
"total_fuel_used": 60443.9,
"location": {
"latitude": 411.372618,
"longitude": -1.254931,
"relative_position": {
"distance": "37",
}
},
"idle_periods": []
},
"relationships": {
"assets": {
"data": [
{
"type": "ABCDFTTG",
"id": "1589799143500003",
"attributes": {
"external_id": "ABCDFTTG",
"hardware_id": "ABCDFTTG"
}
}
]
},
"devices": {
"data": [
{
"type": "ABCDFTTG",
"id": "1585231172900341",
"attributes": {
"serial": "5572016191"
}
},
{
"type": "tablet",
"id": "1587893062600175",
"attributes": {
"serial": "ABCDFTTG"
}
}
]
},
"users": {
"data": [
{
"type": "user",
"id": "ABCDFTTG",
"attributes": {
"external_id": "ABCDFTTG"
}
}
]
}
}
},
"meta": {
"message_id": "11eb-8c75-0b3f87aedbb5",
"consumer_version": "1.2.0",
"origin_version": null,
"timestamp": "2021-06-14T17:42:29Z"
}
}
I want only one row instead of this two. Here is my kusto query which is used for parsing json data into table columns.
Test
|where messageId =="123"
//|mv-expand message=message.data.attributes
|mv-expand message
|mv-expand Value=message.data.relationships.assets.['data']
|mv-expand value_devices=message.data.relationships.devices.['data']
|mv-expand value_user=message.data.relationships.users.['data']
| project type=message.data.type,id=message.data.id,
event=tostring(message.data.attributes.event),
logged_at=tostring(message.data.attributes.logged_at),
distance=toint(message.data.attributes.location.relative_position.distance),
// Value=message.data.relationships.assets.['data'],//.['data']
type_asset=Value.type,asset_id=Value.id,
device_type=value_devices.type,device_id=value_devices.id,
device_attr_serial=value_devices.attributes.serial,
user_type=value_user.type,user_id=value_user.id,
user_external_id=value_user.attributes.external_id
This duplicate row appeared after adding user tag this tag is array so how to handle this array with single id.
I have parse my json data any got the following output.
Expected output should be like
check device_type and device_id columns

How to find match elements in between two collections in mongodb?

I am working on mongodb database, but i am little stuck in one logic, how do i find match elements in between two collections in mongodb.
Users Collection
[{
"_id": "57cd539d168df87ae2695543",
"userid": "3658975589",
"name": "John Doe",
"email": "johndoe#gmail.com",
"number": "123654789"
}, {
"_id": "57cd53e6168df87ae2695544",
"userid": "789456123",
"name": "William Rust",
"email": "williamrust#gmail.com",
"number": "963258741"
}]
Contacts Collection
[{
"_id": "57cd2f6c3966037787ce9550",
"contact": [{
"id": "457899979",
"fullname": "Abcd Hello",
"phonenumber": "123575784565",
"currentUserid": "123456789"
}, {
"id": "7994949849",
"fullname": "Keyboard Mouse",
"phonenumber": "23658974262",
"currentUserid": "123456789"
}, {
"id": "7848848885",
"fullname": "John Doe",
"phonenumber": "852147852",
"currentUserid": "123456789"
}]
}]
So i want to find (phone number) matched elements from these two collections and list out those elements with their name and email.
Please kindly go through my post and suggest me some solution.
I'm guessing that you want to do is "aggregate + lookup". Something like this:
db.users.aggregate([{$lookup:
{
from: "contacts",
localField: "number",
foreignField: "phonenumber",
as: "same"
}
},
{
$match: { "same": { $ne: [] } }
}
])
As a result you get:
{
"_id" : "57cd539d168df87ae2695543",
"userid" : "3658975589",
"name" : "Anshuman Pattnaik",
"email" : "anshuman#gmail.com",
"number" : "7022650603",
"same" : [
{
"_id" : ObjectId("5b361b864aa5144b974c9733"),
"id" : "7848848885",
"fullname" : "Anshuman Pattnaik",
"phonenumber" : "7022650603",
"currentUserid" : "123456789"
}
]
}
If you want show only the name and the email, you have to add { $project: { name: 1, email:1, _id:0 }
db.users.aggregate([{$lookup:
{
from: "contacts",
localField: "number",
foreignField: "phonenumber",
as: "same"
}
},
{
$match: { "same": { $ne: [] } }
},
{ $project: { name: 1, email:1, _id:0 }
])
Then you'll get:
{ "name" : "Anshuman Pattnaik", "email" : "anshuman#gmail.com" }
For this to work you have to correct the insert of your contacts like this:
db.contacts.insert(
[{
"id": "457899979",
"fullname": "Abcd Hello",
"phonenumber": "123575784565",
"currentUserid": "123456789"
}, {
"id": "7994949849",
"fullname": "Keyboard Mouse",
"phonenumber": "23658974262",
"currentUserid": "123456789"
}, {
"id": "7848848885",
"fullname": "Anshuman Pattnaik",
"phonenumber": "7022650603",
"currentUserid": "123456789"
}]
)
Hope it works!
For more information https://docs.mongodb.com/manual/reference/operator/aggregation/lookup/
it's not your complete answer, but it may help you to solve your problem.
you can compare two documents using below function. for more details see this answer
var compareCollections = function(){
db.users collection.find().forEach(function(obj1){
db.contacts collection.find({/*if you know some properties, you can put them here...if don't, leave this empty*/}).forEach(function(obj2){
var equals = function(o1, o2){
// some code.
};
if(equals(ob1, obj2)){
// Do what you want to do
}
});
});
};
db.eval(compareCollections);

Unwind an array in DocumentDB query

I have documents that look like this:
[
{
"id": "e1bb9b05-11f2-459e-37d3-9bf9fed56c96",
"name": "bulbasaur",
"type": [
{
"slot": 2,
"type": {
"url": "https://pokeapi.co/api/v2/type/4/",
"name": "poison"
}
},
{
"slot": 1,
"type": {
"url": "https://pokeapi.co/api/v2/type/12/",
"name": "grass"
}
}
]
}
]
The following query is about as close as I can get, but not quite the output I'm hoping for.
Query
SELECT
c.id, c.name, t.type.name as type
FROM
c
JOIN
t IN c.types
WHERE
c.name = "bulbasaur"
Result
[
{
"id": "e1bb9b05-11f2-459e-37d3-9bf9fed56c96",
"name": "bulbasaur",
"type": "poison"
},
{
"id": "e1bb9b05-11f2-459e-37d3-9bf9fed56c96",
"name": "bulbasaur",
"type": "grass"
}
]
Hoping for
[
{
"id": "e1bb9b05-11f2-459e-37d3-9bf9fed56c96",
"name": "bulbasaur",
"types": ["poison", "grass"]
}
]
Is this possible with a DocumentDB query?
This requires use of DocumentDB UDFs, which can extend query functionality with custom transformations. For example, register this:
function unwindTypeArray(value) {
var result = { id: value.id, name: value.name, types: []};
for (var idx in value.type) {
console.log(idx);
var name = value.type[idx].type.name;
result.types.push(name);
}
return result;
}
Then call it inside a query like:
SELECT udf.unwindTypeArray(c) FROM c WHERE c.name = "bulbasaur"

Transform JSON response with lodash

I'm new in lodash (v3.10.1), and having a hard time understanding.
Hope someone can help.
I have an input something like this:
{
{"id":1,"name":"Matthew","company":{"id":1,"name":"abc","industry":{"id":5,"name":"Medical"}}},
{"id":2,"name":"Mark","company":{"id":1,"name":"abc","industry":{"id":5,"name":"Medical"}}},
{"id":3,"name":"Luke","company":{"id":1,"name":"abc","industry":{"id":5,"name":"Medical"}}},
{"id":4,"name":"John","company":{"id":1,"name":"abc","industry":{"id":5,"name":"Medical"}}},
{"id":5,"name":"Paul","company":{"id":1,"name":"abc","industry":{"id":5,"name":"Medical"}}}
];
I would like to output this or close to this:
{
"industries": [
{
"industry":{
"id":5,
"name":"Medical",
"companies": [
{
"company":{
"id":1,
"name":"abc",
"employees": [
{"id":1,"name":"Matthew"},
{"id":2,"name":"Mark"},
{"id":3,"name":"Luke"},
{"id":4,"name":"John"},
{"id":5,"name":"Paul"}
]
}
}
]
}
}
]
}
Here's something that gets you close to what you want. I structured the output to be an object instead of an array. You don't need the industries or industry properties in your example output. The output structure looks like this:
{
"industry name": {
"id": "id of industry",
"companies": [
{
"company name": "name of company",
"id": "id of company",
"employees": [
{
"id": "id of company",
"name": "name of employee"
}
]
}
]
}
}
I use the _.chain function to wrap the collection with a lodash wrapper object. This enables me to explicitly chain lodash functions.
From there, I use the _.groupBy function to group elements of the collection by their industry name. Since I'm chaining, I don't have to pass in the array again to the function. It's implicitly passed via the lodash wrapper. The second argument of the _.groupBy is the path to the value I want to group elements by. In this case, it's the path to the industry name: company.industry.name. _.groupBy returns an object with each employee grouped by their industry (industries are keys for this object).
I then do use _.transform to transform each industry object. _.transform is essentially _.reduce except that the results returned from the _.transform function is always an object.
The function passed to the _.transform function gets executed against each key/value pair in the object. In the function, I use _.groupBy again to group employees by company. Based off the results of _.groupBy, I map the values to the final structure I want for each employee object.
I then call the _.value function because I want to unwrap the output collection from the lodash wrapper object.
I hope this made sense. If it doesn't, I highly recommend reading Lo-Dash Essentials. After reading the book, I finally got why lodash is so useful.
"use strict";
var _ = require('lodash');
var emps = [
{ "id": 1, "name": "Matthew", "company": { "id": 1, "name": "abc", "industry": { "id": 5, "name": "Medical" } } },
{ "id": 2, "name": "Mark", "company": { "id": 1, "name": "abc", "industry": { "id": 5, "name": "Medical" } } },
{ "id": 3, "name": "Luke", "company": { "id": 1, "name": "abc", "industry": { "id": 5, "name": "Medical" } } },
{ "id": 4, "name": "John", "company": { "id": 1, "name": "abc", "industry": { "id": 5, "name": "Medical" } } },
{ "id": 5, "name": "Paul", "company": { "id": 1, "name": "abc", "industry": { "id": 5, "name": "Medical" } } }
];
var result = _.chain(emps)
.groupBy("company.industry.name")
.transform(function(result, employees, industry) {
result[industry] = {};
result[industry].id = _.get(employees[0], "company.industry.id");
result[ industry ][ 'companies' ] = _.map(_.groupBy(employees, "company.name"), function( employees, company ) {
return {
company: company,
id: _.get(employees[ 0 ], 'company.id'),
employees: _.map(employees, _.partialRight(_.pick, [ 'id', 'name' ]))
};
});
return result;
})
.value();
Results from your example are as follows:
{
"Medical": {
"id": 5,
"companies": [
{
"company": "abc",
"id": 1,
"employees": [
{
"id": 1,
"name": "Matthew"
},
{
"id": 2,
"name": "Mark"
},
{
"id": 3,
"name": "Luke"
},
{
"id": 4,
"name": "John"
},
{
"id": 5,
"name": "Paul"
}
]
}
]
}
}
If you ever wanted the exact same structure as in the questions, I solved it using the jsonata library:
(
/* lets flatten it out for ease of accessing the properties*/
$step1 := $ ~> | $ |
{
"employee_id": id,
"employee_name": name,
"company_id": company.id,
"company_name": company.name,
"industry_id": company.industry.id,
"industry_name": company.industry.name
},
["company", "id", "name"] |;
/* now the magic begins*/
$step2 := {
"industries":
[($step1{
"industry" & $string(industry_id): ${
"id": $distinct(industry_id)#$I,
"name": $distinct(industry_name),
"companies": [({
"company" & $string(company_id): {
"id": $distinct(company_id),
"name": $distinct(company_name),
"employees": [$.{
"id": $distinct(employee_id),
"name": $distinct(employee_name)
}]
}
} ~> $each(function($v){ {"company": $v} }))]
}
} ~> $each(function($v){ {"industry": $v} }))]
};
)
You can see it in action on the live demo site: https://try.jsonata.org/VvW4uTRz_