hello all i am trying to parse an old apache log the output has a correct timestamp but also a #timestamp field, the #timestamp is the now date time, how can i make sure that the timestamp becomes the #timestamp for kibana/elasticsearch.
example input:
172.31.21.26 - - [20/Jul/2017:22:1``0:52 +0200] "GET /mobile/getParent/NzE4MzU1ZmUtNmIwOC00N2JkLTk1YmYtNmNhZTUyZmVmNGYz HTTP/1.1" 200 452 "-" "Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_3 like Mac OS X) AppleWebKit/603.3.8 (KHTML, like Gecko) Mobile/14G60 (4301339520)"
conf file:
input {
file {
path=>"/home/ronald/Downloads/log/httpd/short.log"
start_position=>"beginning"
}
}
filter {
grok {
match => { "message" => "%{COMBINEDAPACHELOG}" }
}
}
output {
elasticsearch{
hosts=>"localhost"
index=>"roha_test"
document_type=>"demo1"
}
stdout{
codec => "rubydebug"
}
}
output:
"request" =>"/mobile/getParent/NzE4MzU1ZmUtNmIwOC00N2JkLTk1YmYtNmNhZTUyZmVmNGYz",
"agent" => "\"Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_3 like Mac OS X) AppleWebKit/603.3.8 (KHTML, like Gecko) Mobile/14G60 (4301339520)\"",
"auth" => "-",
"ident" => "-",
"verb" => "GET",
"message" => "172.31.21.26 - - [20/Jul/2017:22:10:52 +0200] \"GET /mobile/getParent/NzE4MzU1ZmUtNmIwOC00N2JkLTk1YmYtNmNhZTUyZmVmNGYz HTTP/1.1\" 200 452 \"-\" \"Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_3 like Mac OS X) AppleWebKit/603.3.8 (KHTML, like Gecko) Mobile/14G60 (4301339520)\"",
"path" => "/home/ronald/Downloads/log/httpd/short.log",
"referrer" => "\"-\"",
"#timestamp" => 2017-10-06T08:49:10.440Z,
"response" => "200",
"bytes" => "452",
"clientip" => "172.31.21.26",
"#version" => "1",
"host" => "ronald-XPS-13-9343",
"httpversion" => "1.1",
"timestamp" => "20/Jul/2017:22:10:52 +0200"
logstash version 5.6.1
You'll have to add a date filter which converts the timestamp field to a parsed datetime object which elasticsearch understands. Something like:
date {
match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
}
Related
Unfortunately, the events do not work for some WebKit browsers on mobile devices.
There seems to be a problem with the touch-event.
OpenLayers versions: 6.4.3 and 6.5.0
Browsers it does not work with:
Miui Browser 71:
Mozilla/5.0 (Linux; U; Android 10; de-de; Redmi Note 8 Pro Build/QP1A.190711.020) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.141 Mobile Safari/537.36 XiaoMi/MiuiBrowser/12.8.3-gn
Miui Browser 79:
Mozilla/5.0 (Linux; U; Android 10; de-de; Redmi Note 8 Pro Build/QP1A.190711.020) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/79.0.3945.147 Mobile Safari/537.36 XiaoMi/MiuiBrowser/12.10.8-gn
Safari 12:
Mozilla/5.0 (iPad; CPU OS 12_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Mobile/15E148 Safari/604.1
You can try it on this website: https://wp-osm-plugin.hyumika.com/map-with-one-html-popup-marker-in-wordpress/ (you cannot click on the marker, drag or pinch)
With Chrome and Firefox it works, but with the WebKit browsers, I get no console output:
a_MapObj.on('singleclick', function(e) {
console.log('singleclick');
});
a_MapObj.on('click', function(e) {
console.log('click');
});
a_MapObj.on('dblclick', function(e) {
console.log('dblclick');
});
a_MapObj.on('error', function(e) {
console.log('error :' + e);
});
a_MapObj.on('moveend', function(e) {
console.log('moveend');
});
a_MapObj.on('movestart', function(e) {
console.log('movestart');
});
a_MapObj.on('pointermove', function(e) {
console.log('pointermove');
});
a_MapObj.on('pointerdrag', function(e) {
console.log('pointerdrag');
});
Could you please help me to fix this?
Thanks a lot & regards,
Mark
I am trying to build a web crawler using scrapy. I want to change useragent for a single request in the spider. I tried the below code but the user agent is not being updated during the crawl process.
def start_requests(self):
request = Request(
"url",
callback=self.parse_search,
meta={'xpaths': self.xpaths},
headers={
"User-Agent": "Googlebot-Image/1.0"
}
)
return [request]
Your code works perfectly (see my code). But some middleware on your side may affect your User-Agent header:
class UserAgentSpider(scrapy.Spider):
name = 'useragent_spider'
user_agents = [
{'title': 'Galaxy S9', 'value': 'Mozilla/5.0 (Linux; Android 8.0.0; SM-G960F Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36'},
{'title': 'iPhone', 'value': 'Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/69.0.3497.105 Mobile/15E148 Safari/605.1'},
{'title': 'Edge', 'value': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246'},
]
def start_requests(self):
for user_agent in self.user_agents:
yield scrapy.Request(
url="https://www.myip.com/",
headers={
'user-agent': user_agent['value'],
},
cb_kwargs={
'user_agent': user_agent['title']
},
callback=self.parse,
dont_filter=True,
)
def parse(self, response, user_agent):
with open(f"Samples/{user_agent}.htm", 'wb') as f:
f.write(response.body)
If I go to the following web page in Chrome, it loads fine: https://www.cruisemapper.com/?poi=39
However, when I run the following PhantomJS script, which simply goes to the same URL and outputs the entire DOM string to the console, I get a 403 Forbidden message:
var page = require('webpage').create(),
url = 'https://www.cruisemapper.com/?poi=39';
page.open(url, function (status) {
if (status === 'success') {
console.log(page.evaluate(function () {
return document.documentElement.outerHTML;
}));
phantom.exit();
}
});
Here's the exact output to the console:
<html><head>
<title>403 Forbidden</title>
</head><body>
<h1>Forbidden</h1>
<p>You don't have permission to access /
on this server.<br>
</p>
</body></html>
I thought that if I added some sort of user agent string, it might work. As such, I added the following above the console.log line:
page.settings.userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36';
But that didn't work. So then I tried the following instead:
page.customHeaders = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
};
But that didn't work either. Does anyone have any advice on how I can possibly hit up the URL above and not get a 403 Forbidden message? Thank you.
Your code works for me fine (I's suggest viewport size emulation though, see code). If you still get a 403, try changing your IP, it's possible that the site is on to you now (you probably visited that page lots of times).
var page = require('webpage').create(),
url = 'https://www.cruisemapper.com/?poi=39';
page.settings.userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36';
page.viewportSize = { width: 1440, height: 900 }; // <-- otherwise it's 400x300 by default
// It's good to watch for errors on the page
page.onError = function (msg, trace)
{
console.log(msg);
trace.forEach(function(item) {
console.log(' ', item.file, ':', item.line);
})
}
page.open(url, function (status) {
console.log(status);
page.render("page.png"); // Also useful to check if you get what you expect
if (status === 'success') {
console.log(page.evaluate(function () {
return document.documentElement.outerHTML;
}));
phantom.exit();
}
});
here is my problem : Let's say I have some standard Apache logs, like so :
IP1 IP2 - - [13/Jun/2016:14:45:05 +0200] "GET /page/requested.html HTTP/1.1" 200 4860 "-" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0"
I can sucessfully parse these logs with my actual configuration of Logstash :
input {
file {
path => '/home/user/logsDir/*'
}
}
filter {
grok {
match => { "message" => "%{COMBINEDAPACHELOG}"}
}
}
output {
elasticsearch { }
stdout { codec => rubydebug }
}
But on these logs, I apply some machine learning algorithm and I give them a score. So the new log line looks like that :
IP1 IP2 - - [13/Jun/2016:14:45:05 +0200] "GET /page/requested.html HTTP/1.1" 200 4860 "-" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0" 0.00950628507703
Note the 0.00950628507703 at the end of the line, which is the actual score
Now, I would like to parse this line so I could use score for visualisation in Kibana (Logstash is integeated in the whole ELK stack ). So it would be great if the score could be parse as a float.
NB: I can place the score before or after the standard Apache log message and insert any kind of characters between the two (currently it is just a space).
Any idea on how to tackle this problem ?
Thanks in advance !
Eventually I found how to process. I add a little keyword before the score : the word pred
So my lines are know like this :
IP1 IP2 - - [13/Jun/2016:14:45:05 +0200] "GET /page/requested.html HTTP/1.1" 200 4860 "-" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0" pred:0.00950628507703
And I use this configuration for logstash :
input {
file {
path => '/home/user/logsDir/*'
start_position => "beginning"
}
}
filter {
grok {
match => { "message" => "%{COMBINEDAPACHELOG} pred:%{NUMBER:prediction_score}"}
}
# I convert the score into a float in order to vizualise it in Kibana
mutate {
convert => {"prediction_score" => "float"}
}
}
output {
elasticsearch { }
stdout { codec => rubydebug }
}
I hope this will help you if you are stuck with the same problem !
Cheers !
I'm using winston to stream log messages from Express based on various comments elsewhere, my setup is essentially:
var express = require("express"),
winston = require("winston");
// enable web server logging; pipe those log messages through winston
var requestLogger = new (winston.Logger)(
{
transports: [
new (winston.transports.File)(
{
filename: "logs/request.log",
json: false,
timestamp: false
}
)
]
}
),
winstonStream = {
write: function(message, encoding) {
requestLogger.info(message.replace(/(\r?\n)$/, ''));
}
};
this.use(express.logger({stream: winstonStream}));
But I'd like a way to suppress the output of the log level because I know for this particular logger it will always be "info". So rather than:
info: 127.0.0.1 - - [Fri, 20 Sep 2013 13:48:02 GMT] "POST /v1/submission HTTP/1.1" 200 261 "http://localhost:8887/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.65 Safari/537.36"
I would get:
127.0.0.1 - - [Fri, 20 Sep 2013 13:48:02 GMT] "POST /v1/submission HTTP/1.1" 200 261 "http://localhost:8887/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.65 Safari/537.36"