Am getting objects in a s3 buckets using following
s3 = boto3.resource(
service_name='s3',
aws_access_key_id=key_id,
aws_secret_access_key=secret
)
for summary_obj in s3.Bucket(bucket_name).objects.all():
print(summary_obj.key)
Its giving me all object like this
'sub1/sub1_1/file1.zip',
'sub1/sub1_2/file2.zip',
'sub2/sub2_1/file3.zip',
'sub3/file4.zip',
'sub4/sub4_1/file5.zip',
'sub5/sub5_1/file6.zip',
'sub5/sub5_2/file7.zip',
'sub5/sub5_3/file8.zip',
'sub6/'
But i want to have a list of json of all objects with proper directory structure like this to show in my app
[
{'sub1': [
{
'sub1_1': ['file1.zip'] // All files in sub1_1 folder
},
{
'sub1_2': ['file2.zip'] // All files in sub1_2 folder
},
]},
{'sub2': [
{
'sub2_1': [
'file3.zip'
]
}
]},
{'sub3': [
'file4.zip'
]},
{'sub4': [
{
'sub4_1': [
'file5.zip'
]
}
]},
{'sub5': [
{
'sub5_1': [
'file6.zip'
]
},
{
'sub5_2': [
'file7.zip'
]
},
{
'sub5_3': [
'file8.zip'
]
}
]},
{'sub6': []}
]
what is the best way to do this in python3.8?
I give it a try and the closest I could get to your json was through recursion which works with any level of sub-folders and folders:
from collections import defaultdict
objects=['sub1/sub1_1/file1.zip',
'sub1/sub1_2/file2.zip',
'sub2/sub2_1/file3.zip',
'sub3/file4.zip',
'sub4/sub4_1/file5.zip',
'sub5/sub5_1/file6.zip',
'sub5/sub5_2/file7.zip',
'sub5/sub5_3/file8.zip',
'sub5/sub5_3/file9.zip',
'sub5/sub5_3/sub5_4/file1.zip',
'sub5/sub5_3/sub5_4/file2.zip',
'sub6/']
#print(objects)
def construct_dict(in_list, accumulator):
if not in_list:
return
else:
if in_list[0] not in accumulator:
accumulator[in_list[0]] = defaultdict(list)
return construct_dict(in_list[1::], accumulator[in_list[0]])
accumulator = defaultdict(list)
for obj in objects:
construct_dict(obj.split('/'), accumulator)
print(json.dumps(accumulator))
Which gives (the content is same, but structure a bit different):
{
"sub1": {
"sub1_1": {
"file1.zip": {}
},
"sub1_2": {
"file2.zip": {}
}
},
"sub2": {
"sub2_1": {
"file3.zip": {}
}
},
"sub3": {
"file4.zip": {}
},
"sub4": {
"sub4_1": {
"file5.zip": {}
}
},
"sub5": {
"sub5_1": {
"file6.zip": {}
},
"sub5_2": {
"file7.zip": {}
},
"sub5_3": {
"file8.zip": {},
"file9.zip": {},
"sub5_4": {
"file1.zip": {},
"file2.zip": {}
}
}
},
"sub6": {
"": {}
}
}
Related
In my mongodb collection documents are stored in the following format:
{ "_id" : ObjectId("62XXXXXX"), "res" : 12, ... }
{ "_id" : ObjectId("63XXXXXX"), "res" : 23, ... }
{ "_id" : ObjectId("64XXXXXX"), "res" : 78, ... }
...
I need to extract id's for the document for which the value of "res" is outlier (i.e. value < Q1 - 1.5 * IQR or value > Q3 + 1.5 * IQR (Q1, Q3 are percentiles)). I have done this using pandas functionality by retrieving all documents from the collection, which may become slow if the number of documents in collection become too big.
Is there a way to do this using mongodb aggregation pipeline (or just calculating percentiles)?
If I understand how you want to retrieve outliers, here's one way you might be able to do it.
db.collection.aggregate([
{ // partition res into quartiles
"$bucketAuto": {
"groupBy": "$res",
"buckets": 4
}
},
{ // get the max of each quartile
"$group": {
"_id": "$_id.max"
}
},
{ // sort the quartile maxs
"$sort": {
"_id": 1
}
},
{ // put sorted quartile maxs into array
"$group": {
"_id": null,
"maxs": {"$push": "$_id"}
}
},
{ // assign Q1 and Q3
"$project": {
"_id": 0,
"q1": {"$arrayElemAt": ["$maxs", 0]},
"q3": {"$arrayElemAt": ["$maxs", 2]}
}
},
{ // set IQR
"$set": {
"iqr": {
"$subtract": ["$q3", "$q1"]
}
}
},
{ // assign upper/lower outlier thresholds
"$project": {
"outlierThresholdLower": {
"$subtract": [
"$q1",
{"$multiply": ["$iqr", 1.5]}
]
},
"outlierThresholdUpper": {
"$add": [
"$q3",
{"$multiply": ["$iqr", 1.5]}
]
}
}
},
{ // get outlier _id's
"$lookup": {
"from": "collection",
"as": "outliers",
"let": {
"oTL": "$outlierThresholdLower",
"oTU": "$outlierThresholdUpper"
},
"pipeline": [
{
"$match": {
"$expr": {
"$or": [
{"$lt": ["$res", "$$oTL"]},
{"$gt": ["$res", "$$oTU"]}
]
}
}
},
{
"$project": {
"_id": 1
}
}
]
}
}
])
Try it on mongoplayground.net.
One more option based on #rickhg12hs's answer, is to use $setWindowFields:
db.collection.aggregate([
{$setWindowFields: {
sortBy: {res: 1},
output: {
totalCount: {$count: {}},
index: {$sum: 1, window: {documents: ["unbounded", "current"]}}
}
}
},
{$match: {
$expr: {$lte: [
{$abs: {$subtract: [
{$mod: [
{$multiply: [
{$add: ["$index", {$round: {$divide: ["$totalCount", 4]}}]}, 2]},
"$totalCount"
]}, 0]}
}, 1]}
}},
{$group: {_id: null, res: {$push: "$res"}}},
{$project: {_id: 0, q1: {$first: "$res"}, q3: {$last: "$res"},
iqr: {"$subtract": [{$last: "$res"}, {$first: "$res"}]}
}},
{$project: {
outlierThresholdLower: {$subtract: ["$q1", {$multiply: ["$iqr", 1.5]}]},
outlierThresholdUpper: {$add: ["$q3", {$multiply: ["$iqr", 1.5]}]}
}
},
{$lookup: {
from: "collection",
as: "outliers",
let: {oTL: "$outlierThresholdLower", oTU: "$outlierThresholdUpper"},
pipeline: [
{$match: {$expr: {$or: [{$lt: ["$res", "$$oTL"]}, {$gt: ["$res", "$$oTU"]}]}}},
{$project: {_id: 1}}
]
}
}
])
See how it works on the playground example
{
"data":{
"type":"flight-order",
"id":"eJzTd9cPijL1Cg8FAAuUAn0%3D",
"associatedRecords":[
{
"reference":"RZ5JWU",
"creationDate":"2022-01-13T05:40:00.000",
"originSystemCode":"GDS",
"flightOfferId":"1"
}
],
"flightOffers":[
{
"type":"flight-offer",
"id":"1",
"source":"GDS",
"nonHomogeneous":false,
"lastTicketingDate":"2022-03-31",
"itineraries":[
{
"segments":[
{
"departure":{
"iataCode":"ISB",
"at":"2022-03-30T01:40:00"
},
"arrival":{
"iataCode":"DXB",
"terminal":"1",
"at":"2022-03-31T03:55:00"
},
"carrierCode":"PK",
"number":"233",
"aircraft":{
"code":"320"
},
"operating":{
},
"id":"1",
"numberOfStops":0,
"co2Emissions":[
{
"weight":141,
"weightUnit":"KG",
"cabin":"ECONOMY"
}
]
}
]
}
],
"price":{
"currency":"PKR",
"total":"25235.00",
"base":"15190.00",
"fees":[
{
"amount":"0.00",
"type":"TICKETING"
},
{
"amount":"0.00",
"type":"SUPPLIER"
},
{
"amount":"0.00",
"type":"FORM_OF_PAYMENT"
}
],
"grandTotal":"25234.00",
"billingCurrency":"PKR"
},
"pricingOptions":{
"fareType":[
"PUBLISHED"
],
"includedCheckedBagsOnly":true
},
"validatingAirlineCodes":[
"PK"
],
"travelerPricings":[
{
"travelerId":"1",
"fareOption":"STANDARD",
"travelerType":"ADULT",
"price":{
"currency":"PKR",
"total":"25234.00",
"base":"15190.00",
"taxes":[
{
"amount":"5000.00",
"code":"RG"
},
{
"amount":"2000.00",
"code":"SP"
},
{
"amount":"2800.00",
"code":"YD"
},
{
"amount":"244.00",
"code":"ZR"
}
],
"refundableTaxes":"10044.00"
},
"fareDetailsBySegment":[
{
"segmentId":"1",
"cabin":"ECONOMY",
"fareBasis":"VLOWPK",
"class":"V",
"includedCheckedBags":{
"weight":30,
"weightUnit":"KG"
}
}
]
}
]
}
],
"travelers":[
{
"id":"1",
"dateOfBirth":"2003-01-03",
"gender":"FEMALE",
"name":{
"firstName":"Fakhar",
"lastName":"Khan"
},
"documents":[
{
"number":"AG324234234",
"issuanceDate":"2015-01-17",
"expiryDate":"2025-01-17",
"issuanceCountry":"PK",
"issuanceLocation":"Pakistan",
"nationality":"PK",
"documentType":"PASSPORT",
"holder":true
}
],
"contact":{
"purpose":"STANDARD",
"phones":[
{
"deviceType":"MOBILE",
"countryCallingCode":"92",
"number":"3452345678"
}
],
"emailAddress":"hamidafridi.droidor#gmail.com"
}
}
],
"remarks":{
"general":[
{
"subType":"GENERAL_MISCELLANEOUS",
"text":"ONLINE BOOKING FROM INCREIBLE VIAJES"
}
]
},
"ticketingAgreement":{
"option":"DELAY_TO_CANCEL",
"delay":"6D"
},
"contacts":[
{
"addresseeName":{
"firstName":"PABLO RODRIGUEZ"
},
"address":{
"lines":[
"Calle Prado, 16"
],
"postalCode":"28014",
"countryCode":"ES",
"cityName":"Madrid"
},
"purpose":"STANDARD",
"phones":[
{
"deviceType":"LANDLINE",
"countryCallingCode":"34",
"number":"480080071"
},
{
"deviceType":"MOBILE",
"countryCallingCode":"33",
"number":"480080072"
}
],
"companyName":"INCREIBLE VIAJES",
"emailAddress":"support#increibleviajes.es"
}
]
},
"dictionaries":{
"locations":{
"ISB":{
"cityCode":"ISB",
"countryCode":"PK"
},
"DXB":{
"cityCode":"DXB",
"countryCode":"AE"
}
}
}
}
I request for create-order then returned #PNR and now want to issue ticket. #Amadeus
As of now, this Flight Create Orders API allows you to book a flight and generate a PNR, but it does not allow for ticketing. Therefore, one of the requirements in order to use the API in production is to sign a contract with an airline consolidator to issue tickets.
Please check the requirements on the API page. If you want help to find a consolidator get in touch with us via the support channel and we can recommend you one.
Im parsing C++ with ANTLR4 grammar, I have a visitor function for visitDeclarationStatement. In the C++ code that Im trying to parse Person p; or a declaration of any custom type, in the tree I get two similar nodes and I cannot differentiate between the Lvalue and Rvalue!
"declarationStatement": [
{
"blockDeclaration": [
{
"simpleDeclaration": [
{
"declSpecifierSeq": [
{
"declSpecifier": [
{
"typeSpecifier": [
{
"trailingTypeSpecifier": [
{
"simpleTypeSpecifier": [
{
"theTypeName": [
{
"className": [
{
"type": 128,
"text": "Person"
}
]
}
]
}
]
}
]
}
]
}
]
},
{
"declSpecifier": [
{
"typeSpecifier": [
{
"trailingTypeSpecifier": [
{
"simpleTypeSpecifier": [
{
"theTypeName": [
{
"className": [
{
"type": 128,
"text": "p"
}
]
}
]
}
]
}
]
}
]
}
]
}
]
},
{
"type": 124,
"text": ";"
}
]
}
I want to be able to get Variable Type and Variable Name separately. What is the right way of doing that? How can I change the g4 file to get those results in a way that I can differentiate between the type and the name?
Thanks
You don't need to change the grammar. In case of Person p;, which is matched by:
declSpecifierSeq: declSpecifier+ attributeSpecifierSeq?;
the first child of declSpecifierSeq (which is declSpecifier+) will be a List of declSpecifier-contexts, of which the first is the type and the second the name.
My requirement is to generate a line number for every new line generated in the json message. The input message is having array inside array, i.e, parent and child array.
Input message
[
{
id:"1",
Details:[
{
Name:"RAM",
LastName:"Manohar",
DOB:"20-10-1990",
Report:[
{
DateOfJoin:"03-03-2019",
Dept:"HR",
BillCode:"acx-12s",
EffectiveDate:"03-03-2019"
},
{
DateOfJoin:"03-04-2019",
Dept:"HR",
BillCode:"abc-12s",
EffectiveDate:"03-04-2019"
},
{
Name:"Alex",
LastName:"Ham",
DOB:"20-11-1980",
Report:[
{
DateOfJoin:"03-03-2019",
Dept:"HR",
BillCode:"acx-12s",
EffectiveDate:"03-03-2019"
},
{
DateOfJoin:"03-04-2019",
Dept:"HR",
BillCode:"abc-12s",
EffectiveDate:"03-04-2019"
}
]
}
]
},
{
id:"2",
Details:[
{
Name:"Kiran",
LastName:"Kurella",
DOB:"20-10-1980",
Report:[
{
DateOfJoin:"03-03-2019",
Dept:"DC",
BillCode:"acx-12s",
EffectiveDate:"03-03-2019"
},
{
DateOfJoin:"03-04-2019",
Dept:"DC",
BillCode:"abc-12s",
EffectiveDate:"03-04-2019"
},
{
Name:"Sunil",
LastName:"Kumar",
DOB:"20-11-1980",
Report:[
{
DateOfJoin:"03-01-2019",
Dept:"DC",
BillCode:"acx-12s",
EffectiveDate:"03-03-2019"
},
{
DateOfJoin:"03-04-2019",
Dept:"DC",
BillCode:"abc-12s",
EffectiveDate:"03-04-2019"
}
]
}
]
}
]
}
]
}
]
expected output:
[{LineNumber:1,
Dept:"HR",
Name: "Ram"},
{LineNumber:2,
Dept:"HR",
Name: "Alex"},
{LineNumber:3,
Dept:"HR",
Name: "Kiran"},
{LineNumber:4,
Dept:"HR",
Name: "Sunil"}]
Linenumber needs to be generated sequentially and irrespective of parent array or sub array. any help on this will be very appreciated. I have the logic in which i can generate the number using java function but in that case i need to set the variable value (flow variable) inside data weave which can be used in the java function to call recursively.
Use:
payload map {
count: $$
}
ODL version: Carbon
I'm having a problem with getting BGP-LS into the Network Topology. As you can see from below REST output, I set up "bgp-example" and homed to an external eBGP linkstate peer. "effective-rib-in", "adj-rib-in", and "adj-rib-out" all populate - but "loc-rib" does not. For some reason, it is not inheriting the linkstate afi/safi.
I tried debugs for bgp & karaf but saw nothing out of the ordinary (that I could see) - any help would be much appreciated.
thanks
Erik
*bgp configuration
http://192.168.3.42:8181/restconf/config/openconfig-network-instance:network-instances/network-instance/global-bgp/protocols/protocol/openconfig-policy-types:BGP/bgp-example
{
"protocol": [
{
"name": "bgp-example",
"identifier": "openconfig-policy-types:BGP",
"bgp-openconfig-extensions:bgp": {
"global": {
"config": {
"router-id": "192.168.3.42",
"as": 65000
}
},
"neighbors": {
"neighbor": [
{
"neighbor-address": "192.168.3.41",
"config": {
"peer-type": "EXTERNAL",
"peer-as": 65111
},
"afi-safis": {
"afi-safi": [
{
"afi-safi-name": "bgp-openconfig-extensions:LINKSTATE"
}
]
}
}
]
}
}
}
]
}
*loc-rib empty
http://192.168.3.42:8181/restconf/operational/bgp-rib:bgp-rib/rib/bgp-example/loc-rib
{
"loc-rib": {
"tables": [
{
"afi": "bgp-types:ipv4-address-family",
"safi": "bgp-types:unicast-subsequent-address-family",
"bgp-inet:ipv4-routes": {}
}
]
}
}
as you can see, linkstate is making it into every rib, except loc-rib
http://192.168.3.42:8181/restconf/operational/bgp-rib:bgp-rib/rib/bgp-example
{
"rib": [
{
"id": "bgp-example",
"peer": [
{
"peer-id": "bgp://x.x.x.x",
"supported-tables": [
{
"afi": "bgp-types:ipv4-address-family",
"safi": "bgp-types:unicast-subsequent-address-family"
},
{
"afi": "bgp-linkstate:linkstate-address-family",
"safi": "bgp-linkstate:linkstate-subsequent-address-family"
}
],
"effective-rib-in": {
"tables": [
{
"afi": "bgp-linkstate:linkstate-address-family",
"safi": "bgp-linkstate:linkstate-subsequent-address-family",
"bgp-linkstate:linkstate-routes": {
"linkstate-route": [
{
"route-key": "AAMAMAIAAAAAAAAFMgEAABoCAAAEAAD+VwIBAAQAAAAAAgMABgEAFQmQAAEJAAUgCv0YAQ==",
"identifier": 1330,
"advertising-node-descriptors": {
"as-number": 65111,
"domain-id": 0,
"isis-node": {
"iso-system-id": "AQAVCZAA"
}
},
"prefix-descriptors": {
"ip-reachability-information": "x.x.x.x/32"
},
"attributes": {
"origin": {
"value": "igp"
},
"ipv4-next-hop": {
"global": "x.x.x.x"
},
"as-path": {
"segments": [
{
"as-sequence": [
65111
]
}
]
}
},
"protocol-id": "isis-level2"
}
}
rest of output truncated for brevity/readability
OK, figured this out.... turns out I had not enabled LINKSTATE afi/safi in the global config for ODL BGP. I had to DELETE my existing global config, then POST, add neighbors, peers, etc. Now I have the linkstate DB in the loc-rib, AND it's made it to the network topology - BUT - no idea how to view this topology via DLUX....