post a Simple JsonRepresentation in Restlet - restlet

Hi I'm new on RestLet this is a simple example that I want to post a Json representation but after running the client I have error which is mentioned in the below.please help me in this regard.Thank you so much.
The resource :
import org.json.JSONException;
import org.json.JSONObject;
import org.restlet.ext.json.JsonRepresentation;
import org.restlet.resource.Post;
import org.restlet.resource.ServerResource;
import org.restlet.data.Status;
public class DepResource extends ServerResource{
String jsonString="";
#Post
public void acceptJsonRepresentation(JsonRepresentation entity) {
JSONObject json = null;
try {
json = entity.getJsonObject();
// business logic and persistence
String jsonPost=json.toString();
System.out.println(jsonPost);
} catch (JSONException e) {
setStatus(Status.CLIENT_ERROR_BAD_REQUEST);
return;
}
}
}
The client has Error in line 44
import java.io.IOException;
import java.net.InetAddress;
import org.json.JSONException;
import org.json.JSONObject;
import org.restlet.data.CharacterSet;
import org.restlet.data.Method;
import org.restlet.ext.json.JsonRepresentation;
import org.restlet.resource.ClientResource;
public class Client {
public static void main(String[] args) throws JSONException, IOException {
/**##POST Prepration##**/
JSONObject jsonObjectGraph = new JSONObject();
jsonObjectGraph.put("Traffic", 100);
jsonObjectGraph.put("Disksize", 20);
String str=jsonObjectGraph.toString();
JsonRepresentation JRRepDep = new JsonRepresentation(str);
JRRepDep.setCharacterSet(CharacterSet.UTF_8);
System.out.println("with jsonrepresentation: "+JRRepDep.getText());
// TODO Auto-generated method stub
/**********POST**************/
String baseURL1 = "http://" + InetAddress.getLocalHost().getHostAddress() + ":" + "8181";
// Specifying the URL for the resource
String resourceName = "/files";
String ApplicationServerName = baseURL1 + resourceName;
System.out.println("URI at client: " +ApplicationServerName);
// Specifying the REST client and post to REST server
ClientResource restletClient = new ClientResource(ApplicationServerName);
System.out.println(ApplicationServerName);
restletClient.setMethod(Method.POST);
System.out.println("dovomi");
restletClient.post(JRRepDep);
System.out.println("After post");
// Checking the status of the post request
if (restletClient.getStatus().isSuccess())
{
System.out.println("POST Request success.");
restletClient.release();
}
}
}
The server is running without any error
import java.net.InetAddress;
import org.restlet.Application;
import org.restlet.Restlet;
import org.restlet.Server;
import org.restlet.data.Protocol;
import org.restlet.routing.Router;
public class DepServer extends Application {
private static String ipAddress;
private static int port;
public static String getURI()
{
return "http://" + ipAddress + ":" + port;
}
public static void main(String[] args) {
try {
ipAddress = InetAddress.getLocalHost().getHostAddress();
port = 8181;
Server server = new Server(Protocol.HTTP, ipAddress, port);
server.setNext(new DepServer());
server.start();
System.out.print("Server is running");
} catch (Exception e) {
e.printStackTrace();
}
}
#Override
public Restlet createInboundRoot() {
String baseURL = "http://" + ipAddress + ":" + port;
// Create a router restlet.
Router router = new Router(getContext());
// Attach the resources to the router.
router.attach(baseURL + "/files", DepResource.class);
// Return the root router
return router;
}
}
Error:
Jul 20, 2019 2:00:03 PM org.restlet.engine.http.connector.HttpClientHelper start
INFO: Starting the default HTTP client
Exception in thread "main" Internal Server Error (500) - Internal Server Error
at org.restlet.resource.ClientResource.handle(ClientResource.java:870)
at org.restlet.resource.ClientResource.post(ClientResource.java:1209)
at Client.main(Client.java:44)

The question is quite unclear, but if you're asking how to return a JsonRepresentation then you must not return a void but rather a JsonRepresentation see:
public class DepResource extends ServerResource {
String jsonString="{}";
#Post
public JsonRepresentation acceptJsonRepresentation(JsonRepresentation entity) {
// do stuff and return a JsonRepresentation object
Representation representation = new JsonRepresentation(jsonString);
return representation;
}
}

Related

While Sending data over spring webscoket getting NUllpointerException

From last 2 days, I am facing this issue. After debugging, I can see that spring websocket sending data to queue but not received by client
I am not getting any error but sometime i got this error.
Exception in thread "clientInboundChannel-23"
org.springframework.messaging.MessageDeliveryException: Failed to send
message to ExecutorSubscribableChannel[clientOutboundChannel]; nested
exception is java.lang.NullPointerException,
failedMessage=GenericMessage [payload=byte[0],
headers={simpMessageType=DISCONNECT_ACK,
simpDisconnectMessage=GenericMessage [payload=byte[0],
headers={simpMessageType=DISCONNECT, stompCommand=DISCONNECT,
simpSessionAttributes={org.springframework.messaging.simp.SimpAttributes.COMPLETED=true},
simpUser=com.gn.beans.User#44880668, simpSessionId=egw2xot3}],
simpUser=com.gn.beans.User#44880668, simpSessionId=egw2xot3}] at
org.springframework.messaging.support.AbstractMessageChannel.send(AbstractMessageChannel.java:127)
at
org.springframework.messaging.support.AbstractMessageChannel.send(AbstractMessageChannel.java:104)
at
org.springframework.messaging.simp.broker.SimpleBrokerMessageHandler.handleDisconnect(SimpleBrokerMessageHandler.java:328)
at
org.springframework.messaging.simp.broker.SimpleBrokerMessageHandler.handleMessageInternal(SimpleBrokerMessageHandler.java:282)
at
org.springframework.messaging.simp.broker.AbstractBrokerMessageHandler.handleMessage(AbstractBrokerMessageHandler.java:238)
at
org.springframework.messaging.support.ExecutorSubscribableChannel$SendTask.run(ExecutorSubscribableChannel.java:135)
at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.lang.Thread.run(Unknown Source) Caused by:
java.lang.NullPointerException at
com.gn.interceptors.WebSocketInterceptor.preSend(WebSocketInterceptor.java:24)
at
org.springframework.messaging.support.AbstractMessageChannel$ChannelInterceptorChain.applyPreSend(AbstractMessageChannel.java:158)
at
org.springframework.messaging.support.AbstractMessageChannel.send(AbstractMessageChannel.java:113)
... 8 more
My websocket setup is shown below.
WebSocketConfig.java
package com.gn.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.messaging.simp.config.ChannelRegistration;
import org.springframework.messaging.simp.config.MessageBrokerRegistry;
import org.springframework.web.socket.config.annotation.AbstractWebSocketMessageBrokerConfigurer;
import org.springframework.web.socket.config.annotation.EnableWebSocketMessageBroker;
import org.springframework.web.socket.config.annotation.StompEndpointRegistry;
import org.springframework.web.socket.config.annotation.WebSocketTransportRegistration;
import com.gn.interceptors.WebSocketInterceptor;
#Configuration
#EnableWebSocketMessageBroker
public class WebSocketConfig extends AbstractWebSocketMessageBrokerConfigurer {
#Override
public void configureMessageBroker(MessageBrokerRegistry config) {
config.enableSimpleBroker("/advantageClient","/advantageAll");
config.setApplicationDestinationPrefixes("/app");
}
#Override
public void registerStompEndpoints(StompEndpointRegistry registry) {
registry.addEndpoint("/websocket").setAllowedOrigins("*");
registry.addEndpoint("/websocket").setAllowedOrigins("*").withSockJS();
}
#Bean
public WebSocketInterceptor webSocketInterceptor() {
return new WebSocketInterceptor();
}
#Override
public void configureClientInboundChannel(ChannelRegistration registration) {
registration.setInterceptors(webSocketInterceptor());
}
#Override
public void configureClientOutboundChannel(ChannelRegistration registration) {
registration.taskExecutor().corePoolSize(8);
registration.setInterceptors(webSocketInterceptor());
}
#Override
public void configureWebSocketTransport(WebSocketTransportRegistration registration)
{
registration.setSendTimeLimit(15 * 1000)
.setSendBufferSizeLimit(200 * 1024 * 1024)
.setMessageSizeLimit(200 * 1024 * 1024);
}
}
WebSocketUtils.java to send data to client
import java.util.HashMap;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.messaging.simp.SimpMessagingTemplate;
public class WebSocketUtil {
private static final Logger logger = LoggerFactory.getLogger("com.gn.utils");
private static SimpMessagingTemplate template;
public static void setTemplate(SimpMessagingTemplate tmplt) {
template = tmplt;
}
public static void sendToUser(String message,int userId) {
logger.info("Send to user with userID id "+userId+" msg is "+message);
String uId = String.valueOf(userId);
template.convertAndSendToUser(uId,"/advantageClient/connect/", message);
// template.convertAndSendToUser(userId+"","/advantageClient/connect/", message);
}
public static void sendToAll(String message) {
logger.info("Send to all with msg is "+message);
template.convertAndSend("/advantageAll/connect/", message);
}
}
Client Code to connect Websocket
connect() {
const uId = this.homeService.getUserId();
let ws = new SockJS(environment.apiProtocol + '://' + environment.apiHost + ':' + environment.apiPort + UrlConstantsService.urlConstants.advantageService + UrlConstantsService.urlConstants.websocket);
this.ws = Stomp.over(ws);
let that = this;
this.ws.connect({userId: uId}, function(frame) {
that.ws.subscribe('/user/advantageClient/connect/', function(msg) {
that.change.emit(JSON.parse(msg.body));
});
that.ws.subscribe('/advantageAll/connect/', function(msg) {
that.change.emit(JSON.parse(msg.body));
});
}, function(error) {
console.log('error connecting to server', error);
});
}
disconnect() {
if (this.ws !== null) {
this.ws.ws.close();
}
}

How to use camel type converter to convert exchange data into file object in apahce camel? [duplicate]

I am new to Camel and I am facing an issue while sending files to webservice via camel http.
I have a rest web service which consumes Multipart form data type content and accepts input as part of form data.
When I send file and form parameter via camel it gives me the following error at camel console:
Stacktrace
---------------------------------------------------------------------------------------------------------------------------------------
org.apache.camel.component.http.HttpOperationFailedException: HTTP operation failed invoking http://localhost:8080/JAX_RS_Application/resource/restwb/upload with statusCode: 415
at org.apache.camel.component.http.HttpProducer.populateHttpOperationFailedException(HttpProducer.java:230)
at org.apache.camel.component.http.HttpProducer.process(HttpProducer.java:156)
at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61)
at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:129)
at org.apache.camel.management.InstrumentationProcessor.process(InstrumentationProcessor.java:77)
at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:448)
at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:191)
at org.apache.camel.processor.Pipeline.process(Pipeline.java:118)
at org.apache.camel.processor.Pipeline.process(Pipeline.java:80)
at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:191)
at org.apache.camel.component.file.GenericFileConsumer.processExchange(GenericFileConsumer.java:435)
at org.apache.camel.component.file.GenericFileConsumer.processBatch(GenericFileConsumer.java:211)
at org.apache.camel.component.file.GenericFileConsumer.poll(GenericFileConsumer.java:175)
at org.apache.camel.impl.ScheduledPollConsumer.doRun(ScheduledPollConsumer.java:174)
at org.apache.camel.impl.ScheduledPollConsumer.run(ScheduledPollConsumer.java:101)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
The error i get on the server side console is as follows:
SEVERE: MessageBodyReader not found for media type=application/octet-stream, typ
e=class org.glassfish.jersey.media.multipart.FormDataMultiPart, genericType=clas
s org.glassfish.jersey.media.multipart.FormDataMultiPart.
The code snippet of the Rest web-service created via jersey is as follows:
import java.io.IOException;
import java.io.InputStream;
import javax.ws.rs.*;
import javax.ws.rs.core.MediaType;
import org.apache.commons.io.IOUtils;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
import org.glassfish.jersey.media.multipart.FormDataParam;
#Path("/restwb")
public class FileResource {
#POST
#Path("/upload")
#Consumes(MediaType.MULTIPART_FORM_DATA)
public String uploadFile(#FormDataParam("username") String username,#FormDataParam("password") String password,#FormDataParam("upload") InputStream is) {
String output ="Hi "+username+" your password is "+password;
output=output+IOUtils.LINE_SEPARATOR +IOUtils.LINE_SEPARATOR;
output=output+"Output :"+IOUtils.LINE_SEPARATOR+"------------------------------------------------------------------------------"+IOUtils.LINE_SEPARATOR;
try {
output=output+IOUtils.toString(is)+IOUtils.LINE_SEPARATOR+IOUtils.LINE_SEPARATOR;
output=output+"==================================================================================================="+IOUtils.LINE_SEPARATOR+IOUtils.LINE_SEPARATOR;
System.out.println("Output :"+output);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return output;
}
}
And my Camel config is as follows:
import org.apache.camel.*;
import org.apache.camel.builder.RouteBuilder;
import org.apache.camel.impl.DefaultCamelContext;
import org.apache.camel.spi.Synchronization;
import org.apache.camel.spi.UnitOfWork;
import org.apache.http.HttpEntity;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.mime.HttpMultipartMode;
import org.apache.http.entity.mime.MultipartEntityBuilder;
import org.apache.http.entity.mime.content.FileBody;
import org.apache.http.entity.mime.content.StringBody;
import org.apache.james.mime4j.message.Multipart;
import org.apache.log4j.Logger;
import java.io.File;
import java.io.InputStream;
import java.util.List;
import java.util.Map;
/**
* Created by Manish.Pillai on 7/16/2015.
*/
public class LoggingMain {
private static final Logger logger =Logger.getLogger(LoggingMain.class);
public static void main(String[] args) throws Exception{
CamelContext camelContext =new DefaultCamelContext();
try {
camelContext.addRoutes(new RouteBuilder() {
#Override
public void configure() throws Exception {
from("file:C:\\temp?delay=5000&move=processed&moveFailed=error&antExclude=**/processed/**,**/error/**")
.process(new Processor() {
public void process(Exchange exchange) throws Exception {
exchange.getContext().getTypeConverterRegistry().addTypeConverter(HttpEntity.class,InputStream.class,new InputStreamToHttpEntityConvertor());
exchange.getOut().setBody(exchange.getIn().getBody(),HttpEntity.class);
}
})
.to("http://localhost:8080/JAX_RS_Application/resource/restwb/upload");
}
});
camelContext.getRestConfiguration();
camelContext.start();
Thread.sleep(5000);
camelContext.stop();
} catch (Exception e) {
logger.error(e.getMessage());
}
}
static class InputStreamToHttpEntityConvertor implements TypeConverter {
public boolean allowNull() {
return false;
}
public <T> T convertTo(Class<T> type, Object value) throws TypeConversionException {
Exchange exchange=(Exchange)value;
StringBody username = new StringBody("username", ContentType.MULTIPART_FORM_DATA);
StringBody password = new StringBody("password", ContentType.MULTIPART_FORM_DATA);
MultipartEntityBuilder multipartEntityBuilder=MultipartEntityBuilder.create();
multipartEntityBuilder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE);
multipartEntityBuilder.addPart("upload", new FileBody(exchange.getIn().getBody(File.class), ContentType.MULTIPART_FORM_DATA, (String) exchange.getIn().getHeader(Exchange.FILE_NAME)));
multipartEntityBuilder.addPart("username",username);
multipartEntityBuilder.addPart("password",password);
return (T)multipartEntityBuilder.build();
}
public <T> T convertTo(Class<T> aClass, Exchange exchange, Object o) throws TypeConversionException {
return convertTo(aClass,o);
}
public <T> T mandatoryConvertTo(Class<T> type, Object value) throws TypeConversionException, NoTypeConversionAvailableException {
return convertTo(type,value);
}
public <T> T mandatoryConvertTo(Class<T> type, Exchange exchange, Object value) throws TypeConversionException, NoTypeConversionAvailableException {
return convertTo(type,value);
}
public <T> T tryConvertTo(Class<T> type, Object value) {
return convertTo(type,value);
}
public <T> T tryConvertTo(Class<T> type, Exchange exchange, Object value) {
return convertTo(type,value);
}
}
}
Any leads would be helpful.
Well, there are several things that can be improved in your code.
First, since you are using a MultipartEntityBuilder, that means you're using Apache's HttpClient version 4.3+, so for best compatibility you should use Camel's HTTP4 component.
Third, in an example as small as this, you don't really need to use the converter, you can do something like this:
public class LoggingMain {
private static final Logger logger = Logger.getLogger(LoggingMain.class);
public static void main(String[] args) throws Exception {
CamelContext camelContext = new DefaultCamelContext();
try {
camelContext.addRoutes(new RouteBuilder() {
#Override
public void configure() throws Exception {
from("file:C:\\temp?delay=5000&move=processed&moveFailed=error&antExclude=**/processed/**,**/error/**")
.process(new Processor() {
public void process(Exchange exchange) throws Exception {
StringBody username = new StringBody("username", ContentType.MULTIPART_FORM_DATA);
StringBody password = new StringBody("password", ContentType.MULTIPART_FORM_DATA);
MultipartEntityBuilder multipartEntityBuilder = MultipartEntityBuilder.create();
multipartEntityBuilder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE);
multipartEntityBuilder.addPart("username", username);
multipartEntityBuilder.addPart("password", password);
String filename = (String) exchange.getIn().getHeader(Exchange.FILE_NAME);
File file = exchange.getIn().getBody(File.class);
multipartEntityBuilder.addPart("upload", new FileBody(file, ContentType.MULTIPART_FORM_DATA, filename));
exchange.getIn().setBody(multipartEntityBuilder.build());
}
})
.to("http4://localhost:8080/JAX_RS_Application/resource/restwb/upload");
}
});
camelContext.getRestConfiguration();
camelContext.start();
Thread.sleep(5000);
camelContext.stop();
} catch (Exception e) {
logger.error(e.getMessage());
}
}
}
I hope this helps!

Hadoop Record Reader only reads first line then input stream seems to be closed

I'm trying to implement a hadoop job, that counts how often a object (Click) appears in a dataset.
Therefore i wrote a custom file input format. The record reader seems to read only the first line of the given file and the close the input stream.
Here is the code:
The Pojo class:
package model;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.io.WritableComparable;
public class Click implements WritableComparable<Click> {
private String user;
private String clickStart;
private String date;
private String clickTarget;
#Override
public void write(DataOutput out) throws IOException {
out.writeUTF(user);
out.writeUTF(clickStart);
out.writeUTF(date);
out.writeUTF(clickTarget);
}
#Override
public void readFields(DataInput in) throws IOException {
user = in.readUTF();
clickStart = in.readUTF();
date = in.readUTF();
clickTarget = in.readUTF();
}
public int compareTo(Click arg0) {
int response = clickTarget.compareTo(arg0.clickTarget);
if (response == 0) {
response = date.compareTo(arg0.date);
}
return response;
}
public String getUser(String user) {
return this.user;
}
public void setUser(String user) {
this.user = user;
}
public String getClickStart() {
return clickStart;
}
public void setClickStart(String clickStart) {
this.clickStart = clickStart;
}
public String getDate() {
return date;
}
public void setDate(String date) {
this.date = date;
}
public String getClickTarget() {
return clickTarget;
}
public void setClickTarget(String clickTarget) {
this.clickTarget = clickTarget;
}
public String toString() {
return clickStart + "\t" + date;
}
}
Here is the FileInputFormat class:
package ClickAnalysis;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import model.Click;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.util.StringUtils;
import org.apache.tools.ant.types.CommandlineJava.SysProperties;
public class ClickAnalysisInputFormat extends FileInputFormat<Click, IntWritable>{
#Override
public RecordReader<Click, IntWritable> createRecordReader(
InputSplit split, TaskAttemptContext context) throws IOException,
InterruptedException {
System.out.println("Creating Record Reader");
return new ClickReader();
}
public static class ClickReader extends RecordReader<Click, IntWritable> {
private BufferedReader in;
private Click key;
private IntWritable value;
#Override
public void initialize(InputSplit inputSplit, TaskAttemptContext context) throws IOException, InterruptedException {
key = new Click();
value = new IntWritable(1);
System.out.println("Starting to read ...");
FileSplit split = (FileSplit) inputSplit;
Configuration conf = context.getConfiguration();
Path path = split.getPath();
InputStream is = path.getFileSystem(conf).open(path);
in = new BufferedReader(new InputStreamReader(is));
}
#Override
public boolean nextKeyValue() throws IOException, InterruptedException {
String line = in.readLine();
System.out.println("line: " + line);
boolean hasNextKeyValue;
if (line == null) {
System.out.println("line is null");
hasNextKeyValue = false;
} else {
String[] click = StringUtils.split(line, '\\', ';');
System.out.println(click[0].toString());
System.out.println(click[1].toString());
System.out.println(click[2].toString());
key.setClickStart(click[0].toString());
key.setDate(click[1].toString());
key.setClickTarget(click[2].toString());
value.set(1);
System.out.println("done with first line");
hasNextKeyValue = true;
}
System.out.println(hasNextKeyValue);
return hasNextKeyValue;
}
#Override
public Click getCurrentKey() throws IOException, InterruptedException {
return this.key;
}
#Override
public IntWritable getCurrentValue() throws IOException, InterruptedException {
return this.value;
}
#Override
public float getProgress() throws IOException, InterruptedException {
return 0;
}
public void close() throws IOException {
in.close();
System.out.println("in closed");
}
}
}
The Mapper class:
package ClickAnalysis;
import java.io.IOException;
import model.Click;
import model.ClickStartTarget;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.Mapper;
import org.jruby.RubyProcess.Sys;
public class ClickAnalysisMapper extends Mapper<Click, IntWritable, Click, IntWritable> {
private static final IntWritable outputValue = new IntWritable();
#Override
protected void map(Click key, IntWritable value, Context context) throws IOException, InterruptedException {
System.out.println("Key: " + key.getClickStart() + " " + key.getDate() + " " + key.getClickTarget() + " Value: " + value);
outputValue.set(value.get());
System.out.println(outputValue.get());
context.write(key, outputValue);
System.out.println("nach context");
}
}
Partitioner class:
package ClickAnalysis;
import model.Click;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Partitioner;
public class ClickAnalysisPartitioner extends Partitioner<Click, IntWritable> {
#Override
public int getPartition(Click key, IntWritable value, int numPartitions) {
System.out.println("in Partitioner drinnen");
int partition = numPartitions;
return partition;
}
}
Hadoop Job, which is triggered via an Restful web service call in a servlet container, but this shouldn't be the problem:
package ClickAnalysis;
import model.Click;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
public class ClickAnalysisJob {
public int run() throws Exception {
// TODO Auto-generated method stub
Configuration conf = new Configuration();
Job job = Job.getInstance(conf, "ClickAnalysisJob");
job.setJarByClass(ClickAnalysisJob.class);
// Job Input path
FileInputFormat.setInputPaths(job, "hdfs://localhost:9000/user/hadoop/testdata1.csv");
// Job Output path
Path out = new Path("hdfs://localhost:9000/user/hadoop/clickdataAnalysis_out");
FileOutputFormat.setOutputPath(job, out);
out.getFileSystem(conf).delete(out,true);
job.setMapperClass(ClickAnalysisMapper.class);
job.setReducerClass(Reducer.class);
job.setPartitionerClass(ClickAnalysisPartitioner.class);
//job.setReducerClass(ClickAnalysisReducer.class);
job.setInputFormatClass(ClickAnalysisInputFormat.class);
job.setOutputFormatClass(SequenceFileOutputFormat.class);
job.setOutputKeyClass(Click.class);
job.setOutputValueClass(IntWritable.class);
job.setMapOutputKeyClass(Click.class);
job.setMapOutputValueClass(IntWritable.class);
System.out.println("in run drinnen");
//job.setGroupingComparatorClass(ClickTargetAnalysisComparator.class);
job.setNumReduceTasks(1);
int result = job.waitForCompletion(true)? 0:1;
return result;
}
}
Next the dataset (example):
/web/big-data-test-site/test-seite-1;2014-07-08;ein ziel
/web/big-data-test-site/test-seite-1;2014-07-08;ein anderes ziel
/web/big-data-test-site/test-seite-1;2014-07-08;ein anderes ziel
/web/big-data-test-site/test-seite-1;2014-07-08;ein ziel
/web/big-data-test-site/test-seite-1;2014-07-08;ein drittes ziel
/web/big-data-test-site/test-seite-1;2014-07-08;ein ziel
/web/big-data-test-site/test-seite-1;2014-07-08;ein viertes ziel
/web/big-data-test-site/test-seite-1;2014-07-08;ein ziel
When I run the program the syso's are showing following:
in run drinnen
Creating Record Reader
Starting to read ...
line: /web/big-data-test-site/test-seite-1;2014-07-08;ein ziel
/web/big-data-test-site/test-seite-1
2014-07-08
ein ziel
done with first line
true
Key: /web/big-data-test-site/test-seite-1 2014-07-08 ein ziel Value: 1
1
in closed
analyze Method: 1
From that i conclude that the record reader only reads the first line.
Why is this happening and how is it fixed?

Unable to log in to Apache FTP Server

I am trying to integrate the Apache FTP server into my application. I have followed the instructions given here but have run into some problems. Currently I am able to run the server and connect to it from my browser but can not log in. I have tried admin/admin and anonymous/*, but the login fails every time. In the apache-ftpserver-1.0.6 source code I had downloaded, the files associated with the user manager are located in res/conf, although when I try to match that file path in my own program I get an error that says "invalid resource directory name" and am unable to build. I also tried including the files users.properties and ftpd-typical.xml in the main directly and can run, but again cannot log in. It seems like my project does not realize these files are present.
Does anyone have experience with Apache FTP Server that could tell me the correct way to include these files so that I can log in to my server?
Thanks!
P.S. I don't think it should make any difference, but I am developing this program for Android.
In the following code I am crating admin user and non-admin user, setting restrictions of reading, writing and restricting throttling and upload rate limit and imposing download rate limiting.
Added a listener to listen user login and logout download start and download finish events.
import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.ftpserver.FtpServer;
import org.apache.ftpserver.FtpServerFactory;
import org.apache.ftpserver.ftplet.Authority;
import org.apache.ftpserver.ftplet.FileSystemFactory;
import org.apache.ftpserver.ftplet.FtpException;
import org.apache.ftpserver.ftplet.Ftplet;
import org.apache.ftpserver.ftplet.UserManager;
import org.apache.ftpserver.ftpletcontainer.impl.DefaultFtpletContainer;
import org.apache.ftpserver.listener.ListenerFactory;
import org.apache.ftpserver.usermanager.PropertiesUserManagerFactory;
import org.apache.ftpserver.usermanager.SaltedPasswordEncryptor;
import org.apache.ftpserver.usermanager.impl.BaseUser;
import org.apache.ftpserver.usermanager.impl.ConcurrentLoginPermission;
import org.apache.ftpserver.usermanager.impl.TransferRatePermission;
import org.apache.ftpserver.usermanager.impl.WritePermission;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SFTPServer {
// ===========================================================
// Constants
// ===========================================================
private final int FTP_PORT = 2221;
private final String DEFAULT_LISTENER = "default";
// private final Logger LOG = LoggerFactory.getLogger(SFTPServer.class);
private static final List<Authority> ADMIN_AUTHORITIES;
private static final int BYTES_PER_KB = 1024;
private static final String DEFAULT_USER_DIR = "C:\\upload";
public final static int MAX_CONCURRENT_LOGINS = 1;
public final static int MAX_CONCURRENT_LOGINS_PER_IP = 1;
// ===========================================================
// Fields
// ===========================================================
private static FtpServer mFTPServer;
private static UserManager mUserManager;
private static FtpServerFactory mFTPServerFactory;
private ListenerFactory mListenerFactor;
// ===========================================================
// Constructors
// ===========================================================
static {
// Admin Authorities
ADMIN_AUTHORITIES = new ArrayList<Authority>();
ADMIN_AUTHORITIES.add(new WritePermission());
ADMIN_AUTHORITIES.add(new ConcurrentLoginPermission(MAX_CONCURRENT_LOGINS, MAX_CONCURRENT_LOGINS_PER_IP));
ADMIN_AUTHORITIES.add(new TransferRatePermission(Integer.MAX_VALUE, Integer.MAX_VALUE));
}
// ===========================================================
// Getter & Setter
// ===========================================================
// ===========================================================
// Methods for/from SuperClass/Interfaces
// ===========================================================
// ===========================================================
// Methods
// ===========================================================
public void init() throws FtpException {
mFTPServerFactory = new FtpServerFactory();
mListenerFactor = new ListenerFactory();
mListenerFactor.setPort(FTP_PORT);
mFTPServerFactory.addListener(DEFAULT_LISTENER, mListenerFactor.createListener());
mFTPServerFactory.getFtplets().put(FTPLetImpl.class.getName(), new FTPLetImpl());
PropertiesUserManagerFactory userManagerFactory = new PropertiesUserManagerFactory();
userManagerFactory.setFile(new File("ftpusers.properties"));
userManagerFactory.setPasswordEncryptor(new SaltedPasswordEncryptor());
mUserManager = userManagerFactory.createUserManager();
mFTPServerFactory.setUserManager(mUserManager);
this.createAdminUser();
SFTPServer.addUser("admin1", "admin1", 20, 20);
mFTPServer = mFTPServerFactory.createServer();
mFTPServer.start();
}
private UserManager createAdminUser() throws FtpException {
UserManager userManager = mFTPServerFactory.getUserManager();
String adminName = userManager.getAdminName();
if (!userManager.doesExist(adminName)) {
// LOG.info((new
// StringBuilder()).append("Creating user : ").append(adminName).toString());
BaseUser adminUser = new BaseUser();
adminUser.setName(adminName);
adminUser.setPassword(adminName);
adminUser.setEnabled(true);
adminUser.setAuthorities(ADMIN_AUTHORITIES);
adminUser.setHomeDirectory(DEFAULT_USER_DIR);
adminUser.setMaxIdleTime(0);
userManager.save(adminUser);
}
return userManager;
}
public static void addUser(String username, String password, int uploadRateKB, int downloadRateKB) throws FtpException {
BaseUser user = new BaseUser();
user.setName(username);
user.setPassword(password);
user.setHomeDirectory(DEFAULT_USER_DIR);
user.setEnabled(true);
List<Authority> list = new ArrayList<Authority>();
list.add(new TransferRatePermission(downloadRateKB * BYTES_PER_KB, uploadRateKB * BYTES_PER_KB)); // 20KB
list.add(new ConcurrentLoginPermission(MAX_CONCURRENT_LOGINS, MAX_CONCURRENT_LOGINS_PER_IP));
user.setAuthorities(list);
mFTPServerFactory.getUserManager().save(user);
}
public static void restartFTP() throws FtpException {
if (mFTPServer != null) {
mFTPServer.stop();
try {
Thread.sleep(1000 * 3);
} catch (InterruptedException e) {
}
mFTPServer.start();
}
}
public static void stopFTP() throws FtpException {
if (mFTPServer != null) {
mFTPServer.stop();
}
}
public static void pauseFTP() throws FtpException {
if (mFTPServer != null) {
mFTPServer.suspend();
}
}
public static void resumeFTP() throws FtpException {
if (mFTPServer != null) {
mFTPServer.resume();
}
}
public static void main(String... are) {
try {
new SFTPServer().init();
} catch (FtpException e) {
e.printStackTrace();
}
}
// ===========================================================
// Inner and Anonymous Classes
// ===========================================================
}
FTPLET Listener
import java.io.IOException;
import org.apache.ftpserver.ftplet.DefaultFtplet;
import org.apache.ftpserver.ftplet.FtpException;
import org.apache.ftpserver.ftplet.FtpRequest;
import org.apache.ftpserver.ftplet.FtpSession;
import org.apache.ftpserver.ftplet.FtpletResult;
public class FTPLetImpl extends DefaultFtplet {
#Override
public FtpletResult onLogin(FtpSession session, FtpRequest request) throws FtpException, IOException {
System.out.println(session.getUser().getName() + " Logged in");
return super.onLogin(session, request);
}
#Override
public FtpletResult onDisconnect(FtpSession session) throws FtpException, IOException {
System.out.println(session.getUser().getName() + " Disconnected");
return super.onDisconnect(session);
}
#Override
public FtpletResult onDownloadStart(FtpSession session, FtpRequest request) throws FtpException, IOException {
System.out.println(session.getUser().getName() + " Started Downloading File " + request.getArgument());
return super.onDownloadStart(session, request);
}
#Override
public FtpletResult onDownloadEnd(FtpSession session, FtpRequest request) throws FtpException, IOException {
System.out.println("Finished Downloading " + request.getArgument());
return super.onDownloadEnd(session, request);
}
}

Netty 4.0 channel gets null

The setup
Windows 7 Professionnal
Eclipse Juno
Java jre7
Netty 4.0.0 Beta2
I have a netty server running on another machine. Then I have a program running on my machine, which is made to simulate many clients communicating with the server concurrently. In order to do that, I have a thread pool implemented with java.util.concurrent.ExecutorService . Each client creates a thread and submit it to the ExecutorService. Just before it ends, that thread creates another one with the same code. The submited code does those steps :
connect to server by sending a handshake (netty bootstrap A and channel A)
get the token from the handshake response
connect to server (netty bootstrap B and channel B)
send one request to server
receive the response
close the connection
create another thread with the same code
The problem
I sometimes get a NullPointerException in NettySocketCommunication.sendMessage(), on channel.write(byteBuf) when sending a request to the server.
01728 16:25:23.870 [nioEventLoopGroup-3804-2] ERROR
c.f.s.virtualuser.VirtualUser - java.lang.RuntimeException:
java.lang.NullPointerException at
c.f.s.virtualuser.VirtualUser.processMessageStep(VirtualUser.java:324)
at
c.f.s.virtualuser.VirtualUser.processNextStep(VirtualUser.java:252)
at
c.f.s.virtualuser.VirtualUser.onChannelConnected(VirtualUser.java:395)
at
c.f.s.c.m.handler.ClientSocketBasedHandler.channelActive(ClientSocketBasedHandler.java:95)
at
io.netty.channel.DefaultChannelHandlerContext.invokeChannelActive(DefaultChannelHandlerContext.java:774)
at
io.netty.channel.DefaultChannelHandlerContext.fireChannelActive(DefaultChannelHandlerContext.java:760)
at
io.netty.channel.ChannelStateHandlerAdapter.channelActive(ChannelStateHandlerAdapter.java:58)
at
io.netty.channel.DefaultChannelHandlerContext.invokeChannelActive(DefaultChannelHandlerContext.java:774)
at
io.netty.channel.DefaultChannelHandlerContext.fireChannelActive(DefaultChannelHandlerContext.java:760)
at
io.netty.channel.DefaultChannelPipeline.fireChannelActive(DefaultChannelPipeline.java:884)
at
io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:223)
at
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:417)
at
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:365)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:302) at
io.netty.channel.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:110)
at java.lang.Thread.run(Unknown Source)
Caused by:
java.lang.NullPointerException: null at
c.f.s.c.m.NettySocketCommunication.sendMessage(NettySocketCommunication.java:109)
at
c.f.s.virtualuser.VirtualUser.processMessageStep(VirtualUser.java:317)
... 15 common frames omitted
The code
I removed some logging and comments to make the code blocks shorter. I also have the VirtualUser.java class (not shown here) that implements both IVirtualUserCommunication and IVirtualUserMessages interfaces.
[AbstractVirtualUserCommunication.java]
import io.netty.bootstrap.Bootstrap;
import io.netty.channel.Channel;
import io.netty.channel.ChannelHandler;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundMessageHandlerAdapter;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
import io.netty.handler.codec.http.DefaultLastHttpContent;
import io.netty.handler.codec.http.HttpRequestEncoder;
import io.netty.handler.codec.http.HttpResponseDecoder;
import io.netty.util.CharsetUtil;
import [...].Amf3;
import [...].Properties;
import [...].LogFactory;
import [...].Logger;
import [...].AbstractRequest;
import [...].IRouterMessage;
import [...].RouterMessage;
import [...].FrimaHandshake;
import [...].IVirtualUserCommunication;
public abstract class AbstractVirtualUserCommunication implements IVirtualUserCommunication
{
protected static Logger Log = LogFactory.getInstance().getLogger(AbstractVirtualUserCommunication.class);
protected String CONFIG_APPLICATION = "target.server.application";
protected String application;
protected String CONFIG_VERSION = "target.server.version";
protected String version;
protected String CONFIG_HANDSHAKE_PORT = "netty.handshake.port";
protected final int defaultHandshakePort = 80;
// The following variables are used by both HTTP and SOCKET communication
protected Bootstrap bootstrapHandshake; // Netty bootstrap used only for handshake
protected Channel channelHandshake; // Netty channel used only for handshake
protected String token; // The token received through handshake process
// Host & port are set in the connect() method
protected String host;
protected int port;
protected Bootstrap bootstrap; // Netty bootstrap used for communication
protected Channel channel; // Netty channel used for communication
/** Connect to the server to get the token */
public void sendHandshake(String host)
{
// Get properties, with default values if they are not specified
this.application = Properties.getString(CONFIG_APPLICATION, "snowstorm");
this.version = Properties.getString(CONFIG_VERSION, "0.0.1");
int handshakePort = Properties.getInt(CONFIG_HANDSHAKE_PORT, defaultHandshakePort);
bootstrapHandshake = new Bootstrap();
try
{
bootstrapHandshake.group(new NioEventLoopGroup());
bootstrapHandshake.channel(NioSocketChannel.class);
bootstrapHandshake.handler(new HandShakeInitializer(/* this */));
// Connect and listen on handshake host/port
channelHandshake = bootstrapHandshake.connect(host, handshakePort).sync().channel();
channelHandshake.closeFuture().sync();
}
catch (InterruptedException e)
{
Log.error(e);
}
finally
{
bootstrapHandshake.shutdown();
}
}
/** Method called after completion of the handshake (the token has been set). */
protected abstract void afterHandshake();
/** Connect to the target server for stress test script execution. */
protected void connect(ChannelHandler handler)
{
bootstrap = new Bootstrap();
try
{
// Initialize the pipeline
bootstrap.group(new NioEventLoopGroup());
bootstrap.channel(NioSocketChannel.class);
bootstrap.handler(handler);
// Connect and listen on host/port
channel = bootstrap.connect(host, port).sync().channel();
if (channel == null)
{
Log.error("PROBLEM : The channel is null in the afterHandshake() method");
}
channel.closeFuture().sync();
}
catch (InterruptedException e)
{
Log.error(e);
}
finally
{
bootstrap.shutdown();
}
}
/** Create a RouterMessage with the specified request. */
protected IRouterMessage buildMessage(AbstractRequest request)
{
RouterMessage routerMessage = new RouterMessage();
routerMessage.bytes = Amf3.serialize(request);
routerMessage.token = this.token;
routerMessage.application = this.application;
routerMessage.version = this.version;
return routerMessage;
}
#Override
public void disconnect()
{
// TODO Is it dangerous to not call channel.close() ??
if (channel != null)
{
channel.close().awaitUninterruptibly();
}
else
{
Log.error("PROBLEM : The channel is null when calling the disconnect() method");
}
bootstrap.shutdown();
}
#Override
public boolean isConnected()
{
if (channel == null)
{
return false;
}
return channel.isActive();
}
private class HandShakeInitializer extends ChannelInitializer<SocketChannel>
{
public HandShakeInitializer()
{
super();
}
#Override
protected void initChannel(SocketChannel socketChannel) throws Exception
{
socketChannel.pipeline().addLast("encoder", new HttpRequestEncoder());
socketChannel.pipeline().addLast("decoder", new HttpResponseDecoder());
socketChannel.pipeline().addLast("handler", new HandShakeHandler(/* communication */));
}
}
private class HandShakeHandler extends ChannelInboundMessageHandlerAdapter<Object>
{
public HandShakeHandler()
{
super();
}
#Override
public void channelActive(ChannelHandlerContext ctx) throws Exception
{
super.channelActive(ctx);
ctx.write(FrimaHandshake.create(null, version, application));
ctx.flush();
}
#Override
public void messageReceived(ChannelHandlerContext ctx, Object msg) throws Exception
{
if (msg instanceof DefaultLastHttpContent)
{
DefaultLastHttpContent defaultLastHttpContent = (DefaultLastHttpContent) msg;
String content = defaultLastHttpContent.data().toString(CharsetUtil.UTF_8);
// Format = token~publicDNS and we only need the token here
token = content;// .split("~")[0];
Log.debug("Starting a bot with token " + token);
afterHandshake();
}
}
#Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception
{
Log.error(cause);
ctx.close();
}
}
}
[NettySocketCommunication.java]
import io.netty.bootstrap.Bootstrap;
import io.netty.buffer.ByteBuf;
import io.netty.channel.ChannelFuture;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.nio.NioSocketChannel;
import [..­.].AbstractRequest;
import [..­.].IRouterMessage;
import [..­.].Serializer;
import [..­.].ClientSocketBasedHandler;
import [..­.].ClientSocketBasedInitializer;
import [..­.].IVirtualUserMessages;
public class NettySocketCommunication extends AbstractVirtualUserCommunication
{
private ClientSocketBasedHandler handler;
private ChannelFuture testChannelFuture;
public NettySocketCommunication()
{
super();
Log.setLevelToInfo();
this.handler = new ClientSocketBasedHandler();
}
#Override
public void setVirtualUser(IVirtualUserMessages virtualUser)
{
this.handler.setVirtualUser(virtualUser);
}
#Override
public void connect(String host, int port)
{
this.host = host;
this.port = port;
// Get the token from the server through the handshake process
sendHandshake(host);
}
#Override
public boolean connectTest(String host, int port)
{
boolean connectSuccess = false;
bootstrap = new Bootstrap();
// Initialize the pipeline
bootstrap.group(new NioEventLoopGroup());
bootstrap.channel(NioSocketChannel.class);
bootstrap.handler(new ClientSocketBasedInitializer(new ClientSocketBasedHandler()));
// Listen on host/port (connect a channel)
testChannelFuture = bootstrap.connect(host, port);
testChannelFuture.awaitUninterruptibly();
if (testChannelFuture.isSuccess())
{
connectSuccess = true;
}
testChannelFuture.channel().close().awaitUninterruptibly();
bootstrap.shutdown();
return connectSuccess;
}
#Override
protected void afterHandshake()
{
super.connect(new ClientSocketBasedInitializer(handler));
}
#Override
public void sendMessage(AbstractRequest request)
{
IRouterMessage routerMessage = buildMessage(request);
ByteBuf byteBuf = Serializer.encode(routerMessage, true);
// Send message
channel.write(byteBuf);
channel.flush();
}
}
[ClientSocketBasedHandler.java]
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundMessageHandlerAdapter;
import java.util.ArrayList;
import java.util.List;
import [...].Amf3;
import [...].LogFactory;
import [...].Logger;
import [...].IMessage;
import [...].IRouterMessage;
import [...].IVirtualUserMessages;
public class ClientSocketBasedHandler extends ChannelInboundMessageHandlerAdapter<IRouterMessage>
{
protected static Logger Log = LogFactory.getInstance().getLogger(ClientSocketBasedHandler.class);
private IVirtualUserMessages virtualUser;
public ClientSocketBasedHandler()
{
super();
Log.setLevelToInfo();
}
public void setVirtualUser(IVirtualUserMessages virtualUser)
{
this.virtualUser = virtualUser;
}
#Override
public void messageReceived(ChannelHandlerContext ctx, IRouterMessage routerMessage) throws Exception
{
List<IMessage> messages = deserializeMessages(routerMessage.getBytes());
for (IMessage message : messages)
{
Log.debug("Received socket : " + message);
if (virtualUser == null)
{
throw new RuntimeException("Must call the setVirtualUser() method before receiving messages");
}
virtualUser.onManticoreMessageReceived(message);
}
}
protected List<IMessage> deserializeMessages(byte[] bytes)
{
Object probablyMessages = Amf3.deserialize(bytes);
List<IMessage> messages = null;
// List of Messages
if (probablyMessages instanceof ArrayList)
{
messages = (List<IMessage>) probablyMessages;
}
// Single Message
else if (probablyMessages instanceof IMessage)
{
messages = new ArrayList<IMessage>(1);
messages.add((IMessage) probablyMessages);
}
// Probably Pollution
else
{
Log.error("Cannot deserialize message '{}'", probablyMessages.toString());
}
return messages;
}
#Override
public void channelActive(ChannelHandlerContext ctx)
{
if (virtualUser != null)
{
virtualUser.onChannelConnected();
}
}
#Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception
{
Log.error(cause);
ctx.close();
}
}
The search
I looked at netty channels related questions on stack overflow, but couldn't find anything relevant to my case.
Links
http://netty.io/4.0/api/io/netty/channel/Channel.html
http://docs.oracle.com/javase/7/docs/api/java/util/concurrent/ExecutorService.html