AttributeError: 'unicode' object has no attribute 'key' - python-unicode

I'm very new to Python coding and have run into an issue while trying to upgrade some code. I'm working with an app that pulls data via an API from stored data from a scan.
here is the code as it sits working
def _collect_one_host_scan_info(self, host_id, sid, scan_info):
"""
The method to collect all the vulnerabilities of one host and generate the event data.
"""
count = 0
host_uri = self.endpoint + '/' + str(sid) + '/hosts/' + str(host_id)
result = self.client.request(host_uri).get("content")
# if there is exception in request, return None
if result is None:
_LOGGER.info("There is exception in request, return None")
return None
else:
host_info = result.get("info", {})
host_end_time = host_info.get("host_end", "")
if self.ckpt.is_new_host_scan(host_end_time,
self.config.get("start_date")):
self.source = self.url + self.endpoint + '/' + str(
sid) + '/hosts/' + str(host_id)
for vuln in result.get("vulnerabilities", []):
vuln["sid"] = sid
vuln["host_id"] = host_id
#get the port info
plugin_id = vuln.get("plugin_id", "")
port_info = []
if plugin_id:
plugin_uri = "{}/plugins/{}".format(host_uri,
plugin_id)
plugin_outputs = self.client.request(plugin_uri).get(
"content", {}).get("outputs")
ports = []
for output in plugin_outputs:
ports.extend(output.get("ports", {}).keys())
for port in ports:
port_elem = {}
port_items = re.split(r"\s*/\s*", port)
port_elem["port"] = int(port_items[0])
if port_items[1]:
port_elem["transport"] = port_items[1]
if port_items[2]:
port_elem["protocol"] = port_items[2]
port_info.append(port_elem)
vuln = dict(vuln, **scan_info)
vuln = dict(vuln, **host_info)
if port_info:
vuln["ports"] = port_info
entry = NessusObject(
vuln.get("timestamp"), self.sourcetype, self.source,
vuln)
self._print_stream(entry)
count += 1
return count
The data that is being pulled from looks like this
"outputs": [
{
"ports": {
"445 / tcp / cifs": [
{
"hostname": "computer.domain.com"
}
]
},
"has_attachment": 0,
"custom_description": null,
"plugin_output": "\nPath : c:\\program files (x86)\\folder\\bin\\fax.exe\nUsed by services : RFDB\nFile write allowed for groups : Domain Users\nFull control of directory allowed for groups : Domain Users\n\nPath : c:\\program files (x86)\\folder\\bin\\faxrpc.exe\nUsed by services : RFRPC\nFile write allowed for groups : Domain Users\nFull control of directory allowed for groups : Domain Users\n\nPath : c:\\program files (x86)\\folder\\bin\\faxserv.exe\nUsed by services : RFSERVER\nFile write allowed for groups : Domain Users\nFull control of directory allowed for groups : Domain Users\n`,
"hosts": null,
"severity": 3
}
with the working code the return is
ports{}.port 445
ports{}.protocol tcp
ports{}.transport cifs
What I really would like is to grab the "plugin_output" data with the "port" data
I'm currently just trying to replace the "port" data with "plugin_output" data
#get the output info
plugin_id = vuln.get("plugin_id", "")
output_info = []
if plugin_id:
plugin_uri = "{}/plugins/{}".format(host_uri,
plugin_id)
plugin_outputs = self.client.request(plugin_uri).get(
"content", {}).get("outputs")
outputs = []
for output in plugin_outputs:
outputs.extend(output.get("plugin_output", "").keys())
for plugin in plugin_outputs:
plugin_elem = {}
plugin_items = re.split(r"nPath\s*", plugin)
plugin_elem["location1"] = plugin_items[0]
if plugin_items[1]:
plugin_elem["location2"] = plugin_items[1]
if plugin_items[2]:
plugin_elem["location3"] = plugin_items[2]
output_info.append(plugin_elem)
vuln = dict(vuln, **scan_info)
vuln = dict(vuln, **host_info)
if output_info:
vuln["plugin_output"] = output_info
entry = NessusObject(
vuln.get("timestamp"), self.sourcetype, self.source,
vuln)
self._print_stream(entry)
count += 1
what I've done as you can see if just replace the "ports" data with "plugin_output" data and the error received is
AttributeError: 'unicode' object has no attribute key

Well after further efforts I was able to figure out what I needed to do with the code. It was much easier than I thought it would be but sometime when learning a new language its hard to envision what is needed. Code posted below.
def _collect_one_host_scan_info(self, host_id, sid, scan_info):
"""
The method to collect all the vulnerabilities of one host and generate
the event data.
"""
count = 0
host_uri = self.endpoint + '/' + str(sid) + '/hosts/' + str(host_id)
result = self.client.request(host_uri).get("content")
# if there is exception in request, return None
if result is None:
_LOGGER.info("There is exception in request, return None")
return None
else:
host_info = result.get("info", {})
host_end_time = host_info.get("host_end", "")
if self.ckpt.is_new_host_scan(host_end_time,
self.config.get("start_date")):
self.source = self.url + self.endpoint + '/' + str(
sid) + '/hosts/' + str(host_id)
for vuln in result.get("vulnerabilities", []):
vuln["sid"] = sid
vuln["host_id"] = host_id
plugin_id = vuln.get("plugin_id", "")
# get plugin_output data
plugin_output_info = []
if plugin_id:
plugin_uri = "{}/plugins/{}".format(host_uri,
plugin_id)
plugin_outputs = self.client.request(plugin_uri).get(
"content", {}).get("outputs", [])
data_output = []
for output in plugin_outputs:
items = output.get("plugin_output", 'no value')
item = str(items)
#clean = re.sub('[^a-zA-Z0-9-()_*.(:\\)]', ' ', item)
plugin_output_info.append(item)
# get the port info
port_info = []
if plugin_id:
plugin_uri = "{}/plugins/{}".format(host_uri,
plugin_id)
plugin_outputs = self.client.request(plugin_uri).get(
"content", {}).get("outputs", [])
ports = []
for output in plugin_outputs:
ports.extend(output.get("ports", {}).keys())
for port in ports:
port_elem = {}
port_items = re.split(r"\s*/\s*", port)
port_elem["port"] = int(port_items[0])
if port_items[1]:
port_elem["transport"] = port_items[1]
if port_items[2]:
port_elem["protocol"] = port_items[2]
port_info.append(port_elem)
vuln = dict(vuln, **scan_info)
vuln = dict(vuln, **host_info)
if port_info:
vuln["ports"] = port_info
if plugin_output_info:
vuln["plugin_output"] = plugin_output_info
entry = NessusObject(
vuln.get("timestamp"), self.sourcetype, self.source,
vuln)
self._print_stream(entry)
count += 1
return count

Related

How to post an image as a request with Flask Server API in Dart/Flutter?

I want to post a request to a Python-based API that has an image in its body. I have tried to send data with 5 methods:
await http.post()
final api = Uri.parse("https://e8f628d7.ngrok.io/detections");
Map<String, dynamic> body = {'images': image};
final response = await http.post(
api,
body: body,
);
if (response.statusCode == 200) {
final responseJson = json.decode(response.body);
print(responseJson);
}
Client().post()
Map<String, dynamic> body = {'images': image};
var client = new http.Client();
client.post("https://e8f628d7.ngrok.io/detections",body: body).then((response) {
print("Post " + response.statusCode.toString());
});
dio
MultipartRequest
final api = Uri.parse("https://e8f628d7.ngrok.io/detections");
var stream = new http.ByteStream(DelegatingStream.typed(image.openRead()));
var length = await image.length();
var request = new http.MultipartRequest("POST", api);
var multipartFileSign = new http.MultipartFile(
'profile_pic', stream, length,
filename: path.basename(image.path));
request.files.add(multipartFileSign);
// send
var response = await request.send();
print(response.statusCode);
response.stream.transform(utf8.decoder).listen((value) {
print(value);
});
Link of [DELETED]First Answer to this question:
if (image == null) return;
String base64Image = base64Encode(image.readAsBytesSync());
http.post(api, body: {
'images': base64Image,
}).then((res) {
print(res.statusCode);
print(json.decode(res.body));
}).catchError((err) {
print(err);
});
}
I am able to send the image and am getting a 200 success response. But, I am not sure if the image is getting altered or any problem happens while sending the image as the response is empty whereas it should have some sort of response.
This is my app.py from with which my server works:
import time
from absl import app, logging
import cv2
import numpy as np
import tensorflow as tf
from yolov3_tf2.models import (
YoloV3, YoloV3Tiny
)
from yolov3_tf2.dataset import transform_images, load_tfrecord_dataset
from yolov3_tf2.utils import draw_outputs
from flask import Flask, request, Response, jsonify, send_from_directory, abort
import os
# customize your API through the following parameters
classes_path = './data/labels/coco.names'
weights_path = './weights/yolov3.tf'
tiny = False # set to True if using a Yolov3 Tiny model
size = 416 # size images are resized to for model
output_path = './detections/' # path to output folder where images with detections are saved
num_classes = 80 # number of classes in model
# load in weights and classes
physical_devices = tf.config.experimental.list_physical_devices('GPU')
if len(physical_devices) > 0:
tf.config.experimental.set_memory_growth(physical_devices[0], True)
if tiny:
yolo = YoloV3Tiny(classes=num_classes)
else:
yolo = YoloV3(classes=num_classes)
yolo.load_weights(weights_path).expect_partial()
print('weights loaded')
class_names = [c.strip() for c in open(classes_path).readlines()]
print('classes loaded')
# Initialize Flask application
app = Flask(__name__)
# API that returns JSON with classes found in images
#app.route('/detections', methods=['POST'])
def get_detections():
raw_images = []
images = request.files.getlist("images")
image_names = []
for image in images:
image_name = image.filename
image_names.append(image_name)
image.save(os.path.join(os.getcwd(), image_name))
img_raw = tf.image.decode_image(
open(image_name, 'rb').read(), channels=3)
raw_images.append(img_raw)
num = 0
# create list for final response
response = []
for j in range(len(raw_images)):
# create list of responses for current image
responses = []
raw_img = raw_images[j]
num+=1
img = tf.expand_dims(raw_img, 0)
img = transform_images(img, size)
t1 = time.time()
boxes, scores, classes, nums = yolo(img)
t2 = time.time()
print('time: {}'.format(t2 - t1))
print('detections:')
for i in range(nums[0]):
print('\t{}, {}, {}'.format(class_names[int(classes[0][i])],
np.array(scores[0][i]),
np.array(boxes[0][i])))
responses.append({
"class": class_names[int(classes[0][i])],
"confidence": float("{0:.2f}".format(np.array(scores[0][i])*100))
})
response.append({
"image": image_names[j],
"detections": responses
})
img = cv2.cvtColor(raw_img.numpy(), cv2.COLOR_RGB2BGR)
img = draw_outputs(img, (boxes, scores, classes, nums), class_names)
cv2.imwrite(output_path + 'detection' + str(num) + '.jpg', img)
print('output saved to: {}'.format(output_path + 'detection' + str(num) + '.jpg'))
#remove temporary images
for name in image_names:
os.remove(name)
try:
return jsonify({"response":response}), 200
except FileNotFoundError:
abort(404)
# API that returns image with detections on it
#app.route('/image', methods= ['POST'])
def get_image():
image = request.files["images"]
image_name = image.filename
image.save(os.path.join(os.getcwd(), image_name))
img_raw = tf.image.decode_image(
open(image_name, 'rb').read(), channels=3)
img = tf.expand_dims(img_raw, 0)
img = transform_images(img, size)
t1 = time.time()
boxes, scores, classes, nums = yolo(img)
t2 = time.time()
print('time: {}'.format(t2 - t1))
print('detections:')
for i in range(nums[0]):
print('\t{}, {}, {}'.format(class_names[int(classes[0][i])],
np.array(scores[0][i]),
np.array(boxes[0][i])))
img = cv2.cvtColor(img_raw.numpy(), cv2.COLOR_RGB2BGR)
img = draw_outputs(img, (boxes, scores, classes, nums), class_names)
cv2.imwrite(output_path + 'detection.jpg', img)
print('output saved to: {}'.format(output_path + 'detection.jpg'))
# prepare image for response
_, img_encoded = cv2.imencode('.png', img)
response = img_encoded.tostring()
#remove temporary image
os.remove(image_name)
try:
return Response(response=response, status=200, mimetype='image/png')
except FileNotFoundError:
abort(404)
if __name__ == '__main__':
app.run(debug=True, host = '0.0.0.0', port=5000)
I try to send the same image directly through Postman and get the desired response but when I do it with the flutter app, I don't get it. Is there any possibility of the image getting altered or modified? And, is there any other method in which I can send the image to the API other than the above 3?
You need to make sure that you are using a good version of http. There was a regression recently that broke multipart form. It's safest for now to hard code the exact version in pubspec.yaml (You might want to look in pubspec.lock to see what version you were using to confirm that it was one of the ones with the error.)
http: 0.12.0+4
Then try this:
main() async {
http.MultipartRequest request = http.MultipartRequest('POST', Uri.parse(url));
request.files.add(
await http.MultipartFile.fromPath(
'images',
File('kitten1.jpg').path,
contentType: MediaType('application', 'jpeg'),
),
);
http.StreamedResponse r = await request.send();
print(r.statusCode);
print(await r.stream.transform(utf8.decoder).join());
}

Redis | Terraform | Creates every time freshly after execution of terraform apply

I am creating Redis in AWS using Terraform. But When I execute terraform apply command for first time it creates without issues. But If I re-run Terraform apply below TF code destroys the Redis and starts re-creating it instead it should tell me that it already exists start focusing on other newly added resources .
Is it expected behaviour of Redis?
Adding terraform plan in the question:
-/+ resource "aws_elasticache_replication_group" "redis" {
apply_immediately = true
at_rest_encryption_enabled = true
auto_minor_version_upgrade = false
automatic_failover_enabled = true
+ configuration_endpoint_address = (known after apply)
engine = "redis"
engine_version = "5.0.4"
~ id = "dev-af-redis" -> (known after apply)
maintenance_window = "sun:06:00-sun:07:00"
~ member_clusters = [
- "ca-cng-dev-af-redis-001",
- "ca-cng-dev-af-redis-002",
] -> (known after apply)
node_type = "cache.t2.medium"
~ number_cache_clusters = 2 -> (known after apply)
parameter_group_name = "default.redis5.0"
port = 6379
~ primary_endpoint_address = "master.dev-af-redis.qxyj8a.euc1.cache.amazonaws.com" -> (known after apply)
replication_group_description = "Airflow Cluster"
replication_group_id = "dev-af-redis"
security_group_ids = [
"sg-094175ad3062da04d",
]
~ security_group_names = [] -> (known after apply)
- snapshot_retention_limit = 0 -> null
~ snapshot_window = "02:30-03:30" -> (known after apply)
subnet_group_name = "dev-subnet-group-airflow"
tags = {
"Application" = "project"
"BusinessUnit" = "subproject"
"Classification" = "private"
"Environment" = "development"
"Name" = "dev-airflow-redis"
"TechnicalOwner" = "ops"
"Tier" = "orchestration"
}
transit_encryption_enabled = true
+ cluster_mode {
+ num_node_groups = 1
+ replicas_per_node_group = 1 # forces replacement
}
}
Plan: 1 to add, 0 to change, 1 to destroy.
TF code which used to create Redis:-
resource "aws_elasticache_replication_group" "cng_redis" {
replication_group_description = "Cluster"
replication_group_id = "dev-af-redis"
engine = "redis"
engine_version = "5.0.4"
node_type = "cache.t2.medium "
port = 6379
subnet_group_name = "dev-subnet-group-airflow"
security_group_ids = ["${aws_security_group.airflow_sg.id}"]
parameter_group_name = "default.redis5.0"
at_rest_encryption_enabled = true
transit_encryption_enabled = true
maintenance_window = "sun:06:00-sun:07:00"
auto_minor_version_upgrade = false
apply_immediately = true
automatic_failover_enabled = true
cluster_mode {
num_node_groups = "1"
replicas_per_node_group = "1"
}
tags = merge(
var.common_tags,
map("Classification", "private"),
map("Name", "airflow-redis")
)
}
Here is a solution ("this is not a bug, it's a feature" case, I suppose ;) ): https://github.com/terraform-providers/terraform-provider-aws/issues/4817#issuecomment-463993424
I tested it and it works.
You have to add parameter group with cluster-enabled set to yes.
I'm using Redis 5.0.5, so to my aws_elasticache_replication_group I added:
resource "aws_elasticache_replication_group" "elc-rep-group" {
...
automatic_failover_enabled = true #this is required, when cluster-enabled parameter is on
parameter_group_name = "default.redis5.0.cluster.on"
...
}

Slashes in Rally Query

I have a Feature name such as: "This / is / the / name / of my feature ". Rally throws me an error when I try to mention this name as a query string. Is there a way to get around this?
Where do you get the error? I tested this query in WS API, a custom grid and a Ruby script using a feature named "feat/ure"
(Name = feat/ure)
and the query worked in all 3 cases.
Here is the Ruby code that query on a feature with forward slash in the name and assigns a new story to it:
require 'rally_api'
#Setup custom app information
headers = RallyAPI::CustomHttpHeader.new()
headers.name = "My Utility"
headers.vendor = "Nick M RallyLab"
headers.version = "1.0"
# Connection to Rally
config = {:base_url => "https://rally1.rallydev.com/slm"}
config[:username] = "user#co.com"
config[:password] = "secret"
config[:workspace] = "W"
config[:project] = "P"
config[:headers] = headers #from RallyAPI::CustomHttpHeader.new()
#rally = RallyAPI::RallyRestJson.new(config)
obj = {}
obj["Name"] = "new story efd3"
new_s = #rally.create("hierarchicalrequirement", obj)
query = RallyAPI::RallyQuery.new()
query.type = "portfolioitem"
query.fetch = "Name,FormattedID"
query.workspace = {"_ref" => "https://rally1.rallydev.com/slm/webservice/v2.0/workspace/111" }
query.project = {"_ref" => "https://rally1.rallydev.com/slm/webservice/v2.0/project/222" }
query.query_string = "(Name = \"feat/ure\")"
result = #rally.find(query)
feature = result.first
puts feature
field_updates={"PortfolioItem" => feature}
new_s.update(field_updates)
You can also utilize the html replacements for those characters.

How to access project name from a query of type portfolioitem

I am trying to match Project name in my query and also trying to print the name of the project associated with each feature record. I know there are plenty of answers but I couldn't find anything that could help me. I am trying to do something like this:
pi_query.type = "portfolioitem"
pi_query.fetch="Name,FormattedID,Owner,c_ScopingTeam,c_AspirationalRelease,c_AssignedProgram,Tags"
#To be configured as per requirement
pi_query.project_scope_up = false
pi_query.project_scope_down = false
pi_query.order = "FormattedID Asc"
pi_query.query_string = "(Project.Name = \"Uni - Serviceability\")"
pi_results = #rally.find(pi_query)
I am trying to match the project name but it simply doesn't work, I also tried printing the name of the project, i tried Project.Name, Project.Values or simply Project. But it doesn't work. I am guessing it is because of my query type which is "portfolioItem" and I can't change my type because I am getting all other attribute values correctly.
Thanks.
Make sure to fetch Project, e.g: feature_query.fetch = "Name,FormattedID,Project"
and this should work:
feature_query.query_string = "(Project.Name = \"My Project\")"
Here is an example where a feature is found by project name.
require 'rally_api'
#Setup custom app information
headers = RallyAPI::CustomHttpHeader.new()
headers.name = "create story in one project, add it to a feature from another project"
headers.vendor = "Nick M RallyLab"
headers.version = "1.0"
# Connection to Rally
config = {:base_url => "https://rally1.rallydev.com/slm"}
config[:username] = "user#co.com"
config[:password] = "secret"
config[:workspace] = "W"
config[:project] = "Product1"
config[:headers] = headers #from RallyAPI::CustomHttpHeader.new()
#rally = RallyAPI::RallyRestJson.new(config)
obj = {}
obj["Name"] = "new story xyz123"
new_s = #rally.create("hierarchicalrequirement", obj)
query = RallyAPI::RallyQuery.new()
query.type = "portfolioitem"
query.fetch = "Name,FormattedID,Project"
query.workspace = {"_ref" => "https://rally1.rallydev.com/slm/webservice/v2.0/workspace/12352608129" }
query.query_string = "(Project.Name = \"Team Group 1\")"
result = #rally.find(query)
feature = result.first
puts feature
field_updates={"PortfolioItem" => feature}
new_s.update(field_updates)

Quering for Test folder that contains a whitespace using rally_api

I’m doing a tool to create Test folders (test suites) and test cases automatically from a xml file that a Jenkins job provides me.
Probably is a silly thing but I cannot find the solution, the thing is when the test folder name contains a white space, I’m not able to query it to see if already exists. I’ve trid to escape the whitespace and also encode it as url but nothing happens.
test_suite_name = ts_line["name"]
if test_suite_name.match(/\s/)
#test_suite_name_nows = test_suite_name.gsub(/ /,"\\ \\")
test_suite_name_nows = URI::encode(test_suite_name)
end
#==================== Querying the test suite in Rally ==========================
test_suite_query = RallyAPI::RallyQuery.new({:type => :testfolder, :query_string => "(Name = #{test_suite_name_nows})"})
Do you know which should be the format for being able to query test folder names as “Test folder1”??
I always get:
/usr/local/rvm/gems/ruby-2.0.0-p195/gems/rally_api-0.9.14/lib/rally_api/rally_json_connection.rb:153:in `send_request': (StandardError)
Error on request - https://rally1.rallydev.com/slm/webservice/1.42/testfolder.js -
{:errors=>["Could not parse: Cannot parse expression \"Version tests\" as a query"]
Thanks a lot.
Here is the format:
query.query_string = "(Name = \"My Test Folder 1\")"
and a full example:
require 'rally_api'
#Setup custom app information
headers = RallyAPI::CustomHttpHeader.new()
headers.name = "find test folder"
headers.vendor = "NickM RallyLab"
headers.version = "1.0"
# Connection to Rally
config = {:base_url => "https://rally1.rallydev.com/slm"}
config[:username] = "user#co.com"
config[:password] = "1984"
config[:workspace] = "W"
config[:project] = "P"
config[:headers] = headers #from RallyAPI::CustomHttpHeader.new()
#rally = RallyAPI::RallyRestJson.new(config)
query = RallyAPI::RallyQuery.new()
query.type = :test_folder
query.fetch = "Name,FormattedID"
query.workspace = {"_ref" => "https://rally1.rallydev.com/slm/webservice/1.43/workspace/12345.js" } #use yoru OID
query.project = {"_ref" => "https://rally1.rallydev.com/slm/webservice/1.43/project/67890.js" } #user yoru OID
query.page_size = 200 #optional - default is 200
query.limit = 1000 #optional - default is 99999
query.project_scope_up = false
query.project_scope_down = true
query.order = "Name Asc"
query.query_string = "(Name = \"My Test Folder 1\")"
results = #rally.find(query)
results.each do |t|
puts "Name: #{t["Name"]}, FormattedID: #{t["FormattedID"]}"
t.read
#.............
end