I'm trying to use RabbitMQ in an RPC environment where each remote call will take a significant amount of time, producing results continually. I want the results to be delivered to the client as they are generated.
I started with the standard tutorial RPC example, then modified it to use "Direct Reply-to". I publish all the intermediate results back to an "anonymous exclusive callback queue", with out acknowledging the original request. When processing is complete, I send a final message back to the client and then acknowledge the original request. But the client is only seeing the first intermediate message. My client happens to be in PHP and my server is in Python, but I suspect that is not relevant. Does anyone have the magic to make this work? I can post code, but is pretty basic stuff by the cookbook.
Answering my own question. The following worked:
php client:
#!/usr/bin/php
<?php
require_once __DIR__ . '/vendor/autoload.php';
use PhpAmqpLib\Connection\AMQPStreamConnection;
use PhpAmqpLib\Message\AMQPMessage;
class RpcClient {
private $connection;
private $channel;
private $callback_queue;
private $response;
private $corr_id;
public function __construct() {
$this->connection = new AMQPStreamConnection(
'localhost', 5672, 'guest', 'guest'
);
$this->channel = $this->connection->channel();
list($this->callback_queue, ,) = $this->channel->queue_declare(
"", false, false, true, false
);
# For direct reply-to, need to consume amq.rabbitmq.repy-to, a special queue name
# Unclear what happens to the declare above
$this->channel->basic_consume(
$this->callback_queue, '', false, true,
false, false, array($this, 'onResponse')
);
}
# This is going to be called once for each message coming back
public function onResponse($rep) {
if ($rep->get('correlation_id') == $this->corr_id) {
$response = json_decode($rep->body, true);
echo print_r($response['line'], true);
if ($response['type'] == 'final') {
$this->response = $rep->body;
}
}
}
public function call($message_array) {
$this->response = null;
$this->corr_id = uniqid();
$jsonm = json_encode($message_array);
$msg = new AMQPMessage(
$jsonm,
array(
'correlation_id' => $this->corr_id,
### Not sure which of the next two lines is the correct one... if either....
##'reply_to' => 'amq.rabbitmq.reply-to' # This is when using direct reply-to
'reply_to' => $this->callback_queue
)
);
$this->channel->basic_publish($msg, '', 'ansiblePB_rpc_queue');
while (!$this->response) {
$this->channel->wait();
}
return intval($this->response);
}
}
$ansiblepb_rpc = new RpcClient();
$response = $ansiblepb_rpc->call(array('userID' => 'jb1234',
'user_display_name' => 'Joe Bloe',
'limit' => '24000'));
echo ' [.] Got ', $response, "\n";
?>
Python server:
#!/usr/bin/env python
""" 1 """
import glob
import json
import platform
import os
import re
import shutil
import subprocess
import time
import yaml
import pika
class RMQmultireply(object):
""" Generic class to support ansible_playbook on a Rabbit MQ RPC queue"""
def __init__(self, channel, method, props):
#""" Constructor.... duh """
self.channel = channel
self.method = method
self.props = props
def run(self, userID, username, limit):
""" Run the main guts of the service """
cmd = ['/home/dhutchin/devel/rmq/multilineoutput']
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
for line in proc.stdout.readlines():
intermediate_json_result = json.dumps({'type': 'intermediate', 'line': line})
self.channel.basic_publish(exchange='',
routing_key=self.props.reply_to,
properties=pika.BasicProperties(
correlation_id=self.props.correlation_id),
body=str(intermediate_json_result))
#self.channel.basic_ack(delivery_tag=self.method.delivery_tag)
proc.wait()
return proc.returncode
def on_request(channel, method, props, jsonstring):
""" Request has just come in to run ansible_playbook """
playbook = RMQmultireply(channel, method, props)
# fork and exec a playbook
# Recieve each line of output and send them as received back
# to the requestor.
# .run does not return until playbook exits.
# Use "Direct Reply-to" mechanism to return multiple messages to
# our client.
request = yaml.load(jsonstring) # Yes, yaml works better than JSON
returncode = playbook.run(request['userID'], request['user_display_name'], request['limit'])
final_json_result = json.dumps({'type': "final", 'line': '', 'rc': returncode})
channel.basic_publish(exchange='',
routing_key=props.reply_to,
properties=pika.BasicProperties(correlation_id=
props.correlation_id),
body=str(final_json_result))
# Acknowlege the original message so that RabbitMQ can remove it
# from the ansiblePB_rpc_queue queue
channel.basic_ack(delivery_tag=method.delivery_tag)
def main():
""" Its kinda obvious what this does """
try:
connection = pika.BlockingConnection(
pika.ConnectionParameters(host='localhost'))
except Exception:
print "pika.BlockingConnection.... failed... maybe RabbitMQ is not running"
quit()
channel = connection.channel()
channel.queue_declare(queue='ansiblePB_rpc_queue')
channel.basic_qos(prefetch_count=1)
# auto_ack is turned off by default, so we don't need to specify auto_ack=False
channel.basic_consume(queue='ansiblePB_rpc_queue', on_message_callback=on_request)
print " [x] Awaiting RPC requests"
channel.start_consuming()
if __name__ == '__main__':
main()
I am working on two different Symfony 2.8 projects running on different servers. It would like to use compression for faster loading. All resources I found point to mod_deflate. But while the first server does not offer mod_deflate at all, the second server cannot use mod_deflate while FastCGI is enabled.
I only found the information, that one can enable compression within the server (mod_deflate) or "in script". But I did not found any detailed on this "in script" solution.
Is is somehow possible to enable compression in Symfony without using mod_deflate?
You can try to gzip content manually in kernel.response event:
namespace AppBundle\EventListener;
use Symfony\Component\EventDispatcher\EventSubscriberInterface;
use Symfony\Component\HttpKernel\KernelEvents;
use Symfony\Component\HttpKernel\HttpKernelInterface;
class CompressionListener implements EventSubscriberInterface
{
public static function getSubscribedEvents()
{
return array(
KernelEvents::RESPONSE => array(array('onKernelResponse', -256))
);
}
public function onKernelResponse($event)
{
//return;
if ($event->getRequestType() != HttpKernelInterface::MASTER_REQUEST) {
return;
}
$request = $event->getRequest();
$response = $event->getResponse();
$encodings = $request->getEncodings();
if (in_array('gzip', $encodings) && function_exists('gzencode')) {
$content = gzencode($response->getContent());
$response->setContent($content);
$response->headers->set('Content-encoding', 'gzip');
} elseif (in_array('deflate', $encodings) && function_exists('gzdeflate')) {
$content = gzdeflate($response->getContent());
$response->setContent($content);
$response->headers->set('Content-encoding', 'deflate');
}
}
}
And register this listener in config:
app.listener.compression:
class: AppBundle\EventListener\CompressionListener
arguments:
tags:
- { name: kernel.event_subscriber }
i am requesting a webservice using :
use GuzzleHttp\Client;
use GuzzleHttp\Exception\ConnectException;
try {
$client = new Client();
$response = $client->request('GET', $url); //it crashes at this line
$content = json_decode($response->getBody(), true);
}
catch (ConnectException $e) {
\Drupal::logger('amu_hal')->error('incorrect_url'.$url);
}
today the distant server return a error 500.
How can i modify my code not to crash my site when it happens?
I assume that by distant server you mean a server that takes a long time to connect. You can specify a timeout for the request.
Or perhaps the server returned error 500 and it fails during json_decode? You can check the status code returned by the request.
Or even perhaps the code is failing the line that you indicate but the exception ConnectException is not being caught? Try using Exception as a catch-all to debug this situation.
Instead of using Guzzle directly, I recommend that you use the Drupal wrapper (which uses Guzzle under the hood).
$client = Drupal::httpClient();
$request = $client->get($uri, ['connect_timeout' => 5]);
if ($request->getStatusCode() === 200) {
echo 'Connection Success';
} else {
echo sprintf('Error %d occurred', $request->getStatusCode());
}
I have to create the go remote-import server for my project using by apache and I try to create perl module for handler the go get request from user.
The perl module (RemoteImport.pm) was show in below:
sub handler{
my $r = shift;
# check if request is from "go"
return Apache2::Const::DECLINED if ($r->args() !~ /go\-get=1$/);
// Code for provide necessary http body data for go get request and return OK.
}
Apache config:
PerlModule Gerrit::Go::RemoteImport
<Location />
SetHandler perl-script
PerlResponseHandler Gerrit::Go::RemoteImport
</Location>
The go get request can be use normally but I found problem in another http request that have path begin with "/", the Apache does not continue to serve the request as it normally and give not-found page.
Please help me to solve this problem.
I get solution from my team as bellow:
sub go_remote_import_response {
my $r = shift;
# Code for provide necessary http body data for go get request and return OK.
}
sub handler {
my $r = shift;
# check if request is from "go"
return Apache2::Const::OK if ($r->args() !~ /go\-get=1$/);
# change the response handler to our handler if the request are from "go"
$r->handler('perl-script');
$r->set_handlers(PerlResponseHandler => \&go_remote_import_response);
return Apache2::Const::OK;
}
I have apache down checker script (remote server), but I think it doesn't work if httpd has a timeout issue or something similar.
For example, site was offline, but the server was online status.
Should I put timeout stuff or something else? How??
<?php
function GetServerStatus($site, $port)
{
$status = array("OFFLINE", "ONLINE");
$fp = #fsockopen($site, $port, $errno, $errstr, 2);
if (!$fp) {
return $status[0];
} else
{ return $status[1];}
}
?>
<?php
$status = GetServerStatus('xxx.xxx.xxx.xxx',80);
if($status == 'OFFLINE') {
$message = "Server is down now!!";
}
?>
I don't see how this can possibly report incorrectly. You could consider sending a GET / request and timing it out.
You could also consider closing the socket in the success case.