How to open a connection to a mongo database based on a rabbitmq message in nestjs using #golevelup/nestjs-rabbitmq - rabbitmq

I scatered the web for this answer or something similar but with no avail. I did found something that may help explain what am I looking for.
I want to get multiple Mongo connection in a Nestjs project based on a payload received from a RabbitMQ broker. Is there a way to do this in a similar way like in this post with the interception of the request in the declaration of the MongooseModule?
#Module( {
imports: [ RabbitMQModule.forRootAsync( RabbitMQModule, {
imports: [ ConfigModule ],
useClass: AmqpConfig,
inject: [ ConfigService ],
} ) ],
controllers: [ MessagingController ],
providers: [ MessagingService, ConfigService ],
exports: [ MessagingService, RabbitMQModule ]
} )
export class MessagingModule { }
The AmqpConfig class
#Injectable()
export class RabbitConfig {
constructor (
private readonly config: ConfigService,
) { }
createModuleConfig (): RabbitMQConfig | Promise<RabbitMQConfig> {
return {
name: this.name,
uri: this.uri,
exchanges: this.exchanges,
connectionInitOptions: this.connectionInitOptions,
enableControllerDiscovery: true,
};
}
get uri (): string {
const { host, port } = this.config.get( 'amqp' );
return `${ host }:${ port || 5672 }`;
}
get name (): string {
return this.config.get( 'amqp' ).name;
}
get exchanges (): Array<RabbitMQExchangeConfig> {
return Object.values( this.config.get( 'amqp' ).exchanges ) as Array<RabbitMQExchangeConfig>;
}
get connectionInitOptions () {
return this.config.get( 'amqp' ).connectionInitOptions;
}
};
How exactly should I configure the MongooseModule?
#Module( {
imports: [
ConfigModule.forRoot( {
isGlobal: true,
load: [ envConfig ]
} ),
MongooseModule.forRootAsync( {
useClass: MongoConfig,
inject: [ ConfigService ]
} )
],
controllers: [ AppController ],
providers: [ AppService ],
} )
export class AppModule { }
MongoConfig class:
export class MongoConfig implements MongooseOptionsFactory {
constructor ( private readonly config: ConfigService ) { }
createMongooseOptions (): MongooseModuleOptions | Promise<MongooseModuleOptions> {
return {
uri: this.uri
};
}
get uri () {
return this.config.get( 'mongo' ).uri;
}
}

Related

GraphQL pagination partial response with error array

I have a query like below
query {
heroes {
node {
name
}
endCursor
}
}
I am trying to understand how GraphQL can handle the error handling and return partial response. I looked at https://github.com/graphql/dataloader/issues/169 and tried to create a resolver like below;
{
Query: {
heroes: async (_) => {
const heroesData = await loadHeroesFromDataWarehouse();
return {
endCursor: heroesData.endCursor;
node: heroesData.map(h => h.name === 'hulk' ? new ApolloError('Hulk is too powerful') : h)
}
}
}
}
I was hoping it would resolve something like below;
{
"errors": [
{
"message": "Hulk is too powerful",
"path": [
"heroes", "1"
],
}
],
"data": {
"heroes": [
{
"name": "spiderman"
},
null,
{
"name": "ironman"
}
]
}
}
but it is completely failing making the heroes itself null like below;
{
"errors": [
{
"message": "Hulk is too powerful",
"path": [
"heroes"
],
}
],
"data": {
"heroes": null
}
}
How can I make resolver to return me the desired partial response?
Found the solution, basically we need a resolver to resolve the edge model itself;
{
Query: {
heroes: (_) => loadHeroesFromDataWarehouse()
},
HeroesEdge {
node: async (hero) => hero.name === 'hulk' ? new ApolloError('Hulk is too powerful') : hero
}
}

python: create directory structure in Json format from s3 bucket objects

Am getting objects in a s3 buckets using following
s3 = boto3.resource(
service_name='s3',
aws_access_key_id=key_id,
aws_secret_access_key=secret
)
for summary_obj in s3.Bucket(bucket_name).objects.all():
print(summary_obj.key)
Its giving me all object like this
'sub1/sub1_1/file1.zip',
'sub1/sub1_2/file2.zip',
'sub2/sub2_1/file3.zip',
'sub3/file4.zip',
'sub4/sub4_1/file5.zip',
'sub5/sub5_1/file6.zip',
'sub5/sub5_2/file7.zip',
'sub5/sub5_3/file8.zip',
'sub6/'
But i want to have a list of json of all objects with proper directory structure like this to show in my app
[
{'sub1': [
{
'sub1_1': ['file1.zip'] // All files in sub1_1 folder
},
{
'sub1_2': ['file2.zip'] // All files in sub1_2 folder
},
]},
{'sub2': [
{
'sub2_1': [
'file3.zip'
]
}
]},
{'sub3': [
'file4.zip'
]},
{'sub4': [
{
'sub4_1': [
'file5.zip'
]
}
]},
{'sub5': [
{
'sub5_1': [
'file6.zip'
]
},
{
'sub5_2': [
'file7.zip'
]
},
{
'sub5_3': [
'file8.zip'
]
}
]},
{'sub6': []}
]
what is the best way to do this in python3.8?
I give it a try and the closest I could get to your json was through recursion which works with any level of sub-folders and folders:
from collections import defaultdict
objects=['sub1/sub1_1/file1.zip',
'sub1/sub1_2/file2.zip',
'sub2/sub2_1/file3.zip',
'sub3/file4.zip',
'sub4/sub4_1/file5.zip',
'sub5/sub5_1/file6.zip',
'sub5/sub5_2/file7.zip',
'sub5/sub5_3/file8.zip',
'sub5/sub5_3/file9.zip',
'sub5/sub5_3/sub5_4/file1.zip',
'sub5/sub5_3/sub5_4/file2.zip',
'sub6/']
#print(objects)
def construct_dict(in_list, accumulator):
if not in_list:
return
else:
if in_list[0] not in accumulator:
accumulator[in_list[0]] = defaultdict(list)
return construct_dict(in_list[1::], accumulator[in_list[0]])
accumulator = defaultdict(list)
for obj in objects:
construct_dict(obj.split('/'), accumulator)
print(json.dumps(accumulator))
Which gives (the content is same, but structure a bit different):
{
"sub1": {
"sub1_1": {
"file1.zip": {}
},
"sub1_2": {
"file2.zip": {}
}
},
"sub2": {
"sub2_1": {
"file3.zip": {}
}
},
"sub3": {
"file4.zip": {}
},
"sub4": {
"sub4_1": {
"file5.zip": {}
}
},
"sub5": {
"sub5_1": {
"file6.zip": {}
},
"sub5_2": {
"file7.zip": {}
},
"sub5_3": {
"file8.zip": {},
"file9.zip": {},
"sub5_4": {
"file1.zip": {},
"file2.zip": {}
}
}
},
"sub6": {
"": {}
}
}

How to serialize a nest js response with class-transformer while getting data with Typegoose?

I have been trying to work through the NestJs example for the Serialization Section for Mongodb using Typegoose using the class-transformer library. The example given at https://docs.nestjs.com/techniques/serialization only shows how to use serialization in TypeORM. I followed the same process for Typegoose. Here is what I have tried so far.
// cat.domain.ts
import { prop } from '#typegoose/typegoose';
export class Cat {
#prop()
name: string;
#prop()
age: number;
#prop()
breed: string;
}
// cats.service.ts
#Injectable()
export class CatsService {
constructor(
#InjectModel(Cat) private readonly catModel: ReturnModelType<typeof Cat>,
) {}
findAll(): Observable<Cat[]> {
return from(this.catModel.find().exec());
}
findOne(id: string): Observable<Cat> {
return from(this.catModel.findById(id).exec());
}
...
}
// cat.response.ts
import { ObjectId } from 'mongodb';
import { Exclude, Transform } from 'class-transformer';
export class CatResponse {
#Transform(value => value.toString(), { toPlainOnly: true })
_id?: ObjectId;
name: string;
age: number;
#Exclude()
breed: string;
constructor(partial: Partial<CatResponse>) {
Object.assign(this, partial);
}
}
// cats.controller.ts
#Controller('cats')
#UseInterceptors(ClassSerializerInterceptor)
export class CatsController {
constructor(private readonly catsService: CatsService) {}
#Get()
findAll(): Observable<CatResponse[]> {
return this.catsService.findAll();
}
#Get(':id')
findOne(#Param() params: FindOneParamsDto): Observable<CatResponse> {
return this.catsService.findOne(params.id);
}
...
}
I tried running the API call on Get() with id but instead of the breed being excluded from the response I have been getting the following response.
{
"$__": {
"strictMode": true,
"selected": {},
"getters": {},
"_id": {
"_bsontype": "ObjectID",
"id": {
"type": "Buffer",
"data": [
94,
93,
76,
66,
116,
204,
248,
112,
147,
216,
167,
205
]
}
},
"wasPopulated": false,
"activePaths": {
"paths": {
"_id": "init",
"name": "init",
"age": "init",
"breed": "init",
"__v": "init"
},
"states": {
"ignore": {},
"default": {},
"init": {
"_id": true,
"name": true,
"age": true,
"breed": true,
"__v": true
},
"modify": {},
"require": {}
},
"stateNames": [
"require",
"modify",
"init",
"default",
"ignore"
]
},
"pathsToScopes": {},
"cachedRequired": {},
"$setCalled": [],
"emitter": {
"_events": {},
"_eventsCount": 0,
"_maxListeners": 0
},
"$options": {
"skipId": true,
"isNew": false,
"willInit": true
}
},
"isNew": false,
"_doc": {
"_id": {
"_bsontype": "ObjectID",
"id": {
"type": "Buffer",
"data": [
94,
93,
76,
66,
116,
204,
248,
112,
147,
216,
167,
205
]
}
},
"name": "Sylver",
"age": 14,
"breed": "Persian Cat",
"__v": 0
},
"$locals": {},
"$op": null,
"$init": true
}
Can anyone help me with how to serialize response properly?
UPDATE:
class-transformer now works correctly with typegoose, look here for the documentation on how to use it
this is an known issue (#108), typegoose (& mongoose) are incompatible with class-transformer/class-validator
this is because typegoose needs to translate the class into an schema and mongoose will compile it to an model (which isnt the class anymore)
Here is a workaround:
// cats.controller.ts
...
import { classToPlain } from "class-transformer";
...
#Controller('cats')
#UseInterceptors(ClassSerializerInterceptor)
export class CatsController {
constructor(private readonly catsService: CatsService) {}
#Get()
findAll(): Observable<CatResponse[]> {
const cats = this.catsService.findAll();
// transforming the Model to CatResponse class...
const catResponses = cats.map(cat => classToPlain(new CatResponse(cat.toJSON())))
return catResponses;
}
#Get(':id')
findOne(#Param() params: FindOneParamsDto): Observable<CatResponse> {
const cat = this.catsService.findOne(params.id);
const catResponse = classToPlain(new CatResponse(cat.toJSON()));
return
}
...
}
Hope it could help.
For people trying to follow nestjs documentation & using mongoose but ClassSerializerInterceptor not working.
Posting a solution for using class-transformer withe mongoose below which can be helpful for others, it uses custom interceptor that you can see in the nest documentation. https://docs.nestjs.com/interceptors
How the folder structure would be:
src
└── cats
├── dto
│ └── cats-response.dto.ts
├── interceptor
│ └── cats.interceptor.ts
├── schemas
│ └── cat.schema.ts
├── cats.controller.ts
└── cats.service.ts
We will create a dto for cat response called CatsResponseDto in cats-response.dto.ts & in it exclude the breed property from response using Exclude() decorator. By using Dto for response we will create the instance of CatsResponseDto
We will create custom interceptor for cats response called CatsInterceptor in cats.interceptor.ts. You can generate it using nest cli, the command is nest g interceptor cats
cats-response.dto.ts
Create CatsResponseDto to be used in our custom interceptor.
import { Expose, Exclude } from 'class-transformer'
export class CatsResponseDto {
#Expose()
name: string;
#Expose()
age: number;
// Exclude decorator to exclude it from our response.
#Exclude()
breed: string;
}
cats.interceptor.ts
Create out custom CatsInterceptor
Note: plainToClass has been deprecated & now called plainToInstance, I only got to know when used it on my Ide and it prompted. The official documentation yet not updated. https://github.com/typestack/class-transformer#plaintoclass
The changelog does mention about it.
https://github.com/typestack/class-transformer/blob/develop/CHANGELOG.md#041-breaking-change---2021-11-20
import { CallHandler, ExecutionContext, Injectable, NestInterceptor } from '#nestjs/common';
import { plainToInstance } from 'class-transformer'
import { map, Observable } from 'rxjs';
import { CatsResponseDto } from '../dto/cats-response.dto'
#Injectable()
export class CatsInterceptor implements NestInterceptor {
intercept(context: ExecutionContext, handler: CallHandler): Observable<any> {
return handler.handle().pipe(
map((data: any) => {
// run something before the response is sent out.
// Please note that plainToClass is deprecated & is now called plainToInstance
return plainToInstance(CatsResponseDto, data, {
// By using excludeExtraneousValues we are ensuring that only properties decorated with Expose() decorator are included in response.
excludeExtraneousValues: true,
})
})
);
}
}
Our cat schema must have been defined like below (for reference) in cat.schema.ts or similar as per mongoose documentation.
cat.schema.ts
import { Prop, Schema, SchemaFactory } from '#nestjs/mongoose';
import { Document } from 'mongoose';
export type CatDocument = Cat & Document;
#Schema({ timestamps: true })
export class Cat {
// no Id defined here as its automatically added by mongoose unless we explicitly provide option to turn it OFF in schema options.
#Prop({ required: true })
name: string;
#Prop({ required: true })
age: number;
#Prop({ required: true })
breed: string;
}
export const CatSchema = SchemaFactory.createForClass(Cat);
Now bind our custom interceptor CatsInterceptor in cats.controller.ts
cats.controller.ts
import { Cat } from './schemas/cat.schema';
import { CatsInterceptor } from './interceptor/cats.interceptor';
import { CatsService } from './cats.service.ts'
#Controller('cats')
export class CatsController {
constructor(private readonly catsService: CatsService) {}
#Get()
findAll(): Promise<Cat[]> {
return this.catsService.findAll();
}
#UseInterceptors(CatsInterceptor)
#Get(':id')
findOne(#Param() params: FindOneParamsDto): Promise<Cat> {
return this.catsService.findOne(params.id);
}
...
}
RESULT: when calling /cats/{id} response would exclude breed.
related issue:
class serialization not working in nestjs

Convert File in MS Graph API on SPFx return undefined

When i try to download a file from API Graph accesing to Drive or Sites with javascript on SPFx this return undefined.
my webpart code:
import { Version } from '#microsoft/sp-core-library';
import {
BaseClientSideWebPart,
IPropertyPaneConfiguration,
PropertyPaneTextField
} from '#microsoft/sp-webpart-base';
import * as strings from 'Docx2PdfWebPartStrings';
import { MSGraphClient } from '#microsoft/sp-http';
export interface IDocx2PdfWebPartProps {
description: string;
}
export default class Docx2PdfWebPart extends BaseClientSideWebPart<IDocx2PdfWebPartProps> {
public async render(): Promise<void> {
const client: MSGraphClient = await this.context.msGraphClientFactory.getClient();
var tenant = 'test';
var siteID = `${tenant}.sharepoint.com,12adb250-26f4-4dbb-9545-71d029bad763,8fdc3f56-2d6d-42d9-9a4d-d684e73c341e`;
var fileID = '01MBNFB7EIQLARTATNE5G3XDJNYBD2A3IL';
var fileName = 'Test.docx';
//This work
var site = await client.api(`/sites/${tenant}.sharepoint.com:/sites/dev:/drive?$select=id,weburl`).get();
console.log(site);
try {
//This not work
var fileFromDrive = await client.api(`/drive/root:/${fileName}:/content?format=pdf`).get();
console.log(fileFromDrive);
var fileFromSite = await client.api(`/sites/${siteID}/drive/items/${fileID}/content?format=pdf`).get();
console.log(fileFromSite);
} catch (error) {
console.log(error);
}
this.domElement.innerHTML = `<h1>Hola Mundo</h1>`;
}
protected get dataVersion(): Version {
return Version.parse('1.0');
}
protected getPropertyPaneConfiguration(): IPropertyPaneConfiguration {
return {
pages: [
{
header: {
description: strings.PropertyPaneDescription
},
groups: [
{
groupName: strings.BasicGroupName,
groupFields: [
PropertyPaneTextField('description', {
label: strings.DescriptionFieldLabel
})
]
}
]
}
]
};
}
}
The chrome console log
But when i use Graph Explorer it works correctly
This is my package-solution.json
{
"$schema": "https://developer.microsoft.com/json-schemas/spfx-build/package-solution.schema.json",
"solution": {
"name": "docx-2-pdf-client-side-solution",
"id": "f4b5db4f-d9ff-463e-b62e-0cc9c9e94089",
"version": "1.0.0.0",
"includeClientSideAssets": true,
"skipFeatureDeployment": true,
"isDomainIsolated": false,
"webApiPermissionRequests": [
{
"resource": "Microsoft Graph",
"scope": "Sites.Read.All"
},
{
"resource": "Microsoft Graph",
"scope": "Files.Read.All"
},
{
"resource": "Microsoft Graph",
"scope": "Files.ReadWrite.All"
},
{
"resource": "Microsoft Graph",
"scope": "Sites.ReadWrite.All"
}
]
},
"paths": {
"zippedPackage": "solution/docx-2-pdf.sppkg"
}
}
I use the following articles
https://learn.microsoft.com/en-us/graph/api/driveitem-get-content?view=graph-rest-1.0&tabs=javascript
https://learn.microsoft.com/en-us/graph/api/driveitem-get-content-format?view=graph-rest-1.0&tabs=javascript#code-try-1
Try using the callback property instead of await:
client.api(`/drive/root:/${fileName}:/content?format=pdf`).get((err, response) => console.log("your response:", err, response));

Elastic Search when to add dynamic mappings

I've been having troubles with Elastic Search (ES) dynamic mappings. Seems like I'm in a catch-22. https://www.elastic.co/guide/en/elasticsearch/guide/current/custom-dynamic-mapping.html
The main goal is to store everything as a string that comes into ES.
What I've tried:
In ES you can't create a dynamic mapping until the index has been
created. Okay, makes sense.
I can't create an empty index, so if
the first item sent into the index is not a string, I can't
re-assign it... I won't know what type of object with be the first
item in the index, it could be any type, due to how the the app accepts a variety of objects/events.
So if I can't create the mapping ahead of time, and I can't insert an empty index to create the mapping, and I can't change the mapping after the fact, how do I deal with the first item if its NOT a string???
Here's what I'm currently doing (using the Javascript Client).
createESIndex = function (esClient){
esClient.index({
index: 'timeline-2015-11-21',
type: 'event',
body: event
},function (error, response) {
if (error) {
logger.log(logger.SEVERITY.ERROR, 'acceptEvent elasticsearch create failed with: '+ error + " req:" + JSON.stringify(event));
console.log(logger.SEVERITY.ERROR, 'acceptEvent elasticsearch create failed with: '+ error + " req:" + JSON.stringify(event));
res.status(500).send('Error saving document');
} else {
res.status(200).send('Accepted');
}
});
}
esClientLookup.getClient( function(esClient) {
esClient.indices.putTemplate({
name: "timeline-mapping-template",
body:{
"template": "timeline-*",
"mappings": {
"event": {
"dynamic_templates": [
{ "timestamp-only": {
"match": "#timestamp",
"match_mapping_type": "date",
"mapping": {
"type": "date",
}
}},
{ "all-others": {
"match": "*",
"match_mapping_type": "string",
"mapping": {
"type": "string",
}
}
}
]
}
}
}
}).then(function(res){
console.log("put template response: " + JSON.stringify(res));
createESIndex(esClient);
}, function(error){
console.log(error);
res.status(500).send('Error saving document');
});
});
Index templates to the rescue !! That's exactly what you need, the idea is to create a template of your index and as soon as you wish to store a document in that index, ES will create it for you with the mapping you gave (even dynamic ones)
curl -XPUT localhost:9200/_template/my_template -d '{
"template": "index_name_*",
"settings": {
"number_of_shards": 1
},
"mappings": {
"type_name": {
"dynamic_templates": [
{
"strings": {
"match": "*",
"match_mapping_type": "*",
"mapping": {
"type": "string"
}
}
}
],
"properties": {}
}
}
}'
Then when you index anything in an index whose name matches index_name_*, the index will be created with the dynamic mapping above.
For instance:
curl -XPUT localhost:9200/index_name_1/type_name/1 -d '{
"one": 1,
"two": "two",
"three": true
}'
That will create a new index called index_name_1 with a mapping type for type_name where all properties are string. You can verify that with
curl -XGET localhost:9200/index_name_1/_mapping/type_name
Response:
{
"index_name_1" : {
"mappings" : {
"type_name" : {
"dynamic_templates" : [ {
"strings" : {
"mapping" : {
"type" : "string"
},
"match" : "*",
"match_mapping_type" : "*"
}
} ],
"properties" : {
"one" : {
"type" : "string"
},
"three" : {
"type" : "string"
},
"two" : {
"type" : "string"
}
}
}
}
}
}
Note that if you're willing to do this via the Javascript API, you can use the indices.putTemplate call.
export const user = {
email: {
type: 'text',
},
};
export const activity = {
date: {
type: 'text',
},
};
export const common = {
name: {
type: 'text',
},
};
import { Client } from '#elastic/elasticsearch';
import { user } from './user';
import { activity } from './activity';
import { common } from './common';
export class UserDataFactory {
private schema = {
...user,
...activity,
...common,
relation_type: {
type: 'join',
eager_global_ordinals: true,
relations: {
parent: ['activity'],
},
},
};
constructor(private client: Client) {
Object.setPrototypeOf(this, UserDataFactory.prototype);
}
async create() {
const settings = {
settings: {
analysis: {
normalizer: {
useLowercase: {
filter: ['lowercase'],
},
},
},
},
mappings: {
properties: this.schema,
},
};
const { body } = await this.client.indices.exists({
index: ElasticIndex.UserDataFactory,
});
await Promise.all([
await (async (client) => {
await new Promise(async function (resolve, reject) {
if (!body) {
await client.indices.create({
index: ElasticIndex.UserDataFactory,
});
}
resolve({ body });
});
})(this.client),
]);
await this.client.indices.close({ index: ElasticIndex.UserDataFactory });
await this.client.indices.putSettings({
index: ElasticIndex.UserDataFactory,
body: settings,
});
await this.client.indices.open({
index: ElasticIndex.UserDataFactory,
});
await this.client.indices.putMapping({
index: ElasticIndex.UserDataFactory,
body: {
dynamic: 'strict',
properties: {
...this.schema,
},
},
});
}
}
wrapper.ts
class ElasticWrapper {
private _client: Client = new Client({
node: process.env.elasticsearch_node,
auth: {
username: 'elastic',
password: process.env.elasticsearch_password || 'changeme',
},
ssl: {
ca: process.env.elasticsearch_certificate,
rejectUnauthorized: false,
},
});
get client() {
return this._client;
}
}
export const elasticWrapper = new ElasticWrapper();
index.ts
new UserDataFactory(elasticWrapper.client).create();