How to use Google QPX API with Java for flight search? - api

I am trying to build a simple Java code that obtains flight options using Google QPX API, for the flight from New York to London. I signed up with Google and got API_key.
I read the documentation, but unfortunately, I couldn't find any example that show me how to do it.
Here what I tried so far:
import com.google.api.services.qpxExpress.model.*;
import java.util.*;
public class Main
{
public static void main(String[] args)
{
// Passengers
PassengerCounts passengers = new PassengerCounts();
passengers.setAdultCount(2);
// Slice
List<SliceInput> slices = new ArrayList<SliceInput>();
SliceInput slice = new SliceInput();
slice.setOrigin("JFK"); // John Kennedy Airport in Ney York
slice.setDestination("LHR"); // London Heathrow
slice.setDate("2015-07-01");
slices.add(slice);
// Options request
TripOptionsRequest tripOptions = new TripOptionsRequest();
tripOptions.setPassengers(passengers);
tripOptions.setSlice(slices);
// Search request
TripsSearchRequest tripSearchReq = new TripsSearchRequest();
tripSearchReq.setRequest(tripOptions);
// Next steps?
// Setting up QPXExpress?
}
}
I will appreciate it if someone can help me to complete the code.
Thanks in advance.

This is a simple code, successfully tested, hope it helps
package com.airline.api;
import java.io.IOException;
import java.util.*;
import javax.swing.text.View;
import com.google.api.client.auth.oauth2.Credential;
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.services.qpxExpress.QPXExpressRequestInitializer;
import com.google.api.services.qpxExpress.QPXExpress;
import com.google.api.services.qpxExpress.model.FlightInfo;
import com.google.api.services.qpxExpress.model.LegInfo;
import com.google.api.services.qpxExpress.model.PassengerCounts;
import com.google.api.services.qpxExpress.model.PricingInfo;
import com.google.api.services.qpxExpress.model.SegmentInfo;
import com.google.api.services.qpxExpress.model.SliceInfo;
import com.google.api.services.qpxExpress.model.TripOption;
import com.google.api.services.qpxExpress.model.TripOptionsRequest;
import com.google.api.services.qpxExpress.model.TripsSearchRequest;
import com.google.api.services.qpxExpress.model.SliceInput;
import com.google.api.services.qpxExpress.model.TripsSearchResponse;
public class AirlineReservation {
/**
* #param args
*/
private static final String APPLICATION_NAME = "MyFlightApplication";
private static final String API_KEY = "AIzaSyDnBCdsmTnrL5XFrO2TjJyvFioswjakNYU";
/** Global instance of the HTTP transport. */
private static HttpTransport httpTransport;
/** Global instance of the JSON factory. */
private static final JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance();
public static void main(String[] args) {
// TODO Auto-generated method stub
try {
httpTransport = GoogleNetHttpTransport.newTrustedTransport();
PassengerCounts passengers= new PassengerCounts();
passengers.setAdultCount(1);
List<SliceInput> slices = new ArrayList<SliceInput>();
SliceInput slice = new SliceInput();
slice.setOrigin("NYC");
slice.setDestination("LGA");
slice.setDate("2015-04-29");
slices.add(slice);
TripOptionsRequest request= new TripOptionsRequest();
request.setSolutions(10);
request.setPassengers(passengers);
request.setSlice(slices);
TripsSearchRequest parameters = new TripsSearchRequest();
parameters.setRequest(request);
QPXExpress qpXExpress= new QPXExpress.Builder(httpTransport, JSON_FACTORY, null).setApplicationName(APPLICATION_NAME)
.setGoogleClientRequestInitializer(new QPXExpressRequestInitializer(API_KEY)).build();
TripsSearchResponse list= qpXExpress.trips().search(parameters).execute();
List<TripOption> tripResults=list.getTrips().getTripOption();
String id;
for(int i=0; i<tripResults.size(); i++){
//Trip Option ID
id= tripResults.get(i).getId();
System.out.println("id "+id);
//Slice
List<SliceInfo> sliceInfo= tripResults.get(i).getSlice();
for(int j=0; j<sliceInfo.size(); j++){
int duration= sliceInfo.get(j).getDuration();
System.out.println("duration "+duration);
List<SegmentInfo> segInfo= sliceInfo.get(j).getSegment();
for(int k=0; k<segInfo.size(); k++){
String bookingCode= segInfo.get(k).getBookingCode();
System.out.println("bookingCode "+bookingCode);
FlightInfo flightInfo=segInfo.get(k).getFlight();
String flightNum= flightInfo.getNumber();
System.out.println("flightNum "+flightNum);
String flightCarrier= flightInfo.getCarrier();
System.out.println("flightCarrier "+flightCarrier);
List<LegInfo> leg=segInfo.get(k).getLeg();
for(int l=0; l<leg.size(); l++){
String aircraft= leg.get(l).getAircraft();
System.out.println("aircraft "+aircraft);
String arrivalTime= leg.get(l).getArrivalTime();
System.out.println("arrivalTime "+arrivalTime);
String departTime=leg.get(l).getDepartureTime();
System.out.println("departTime "+departTime);
String dest=leg.get(l).getDestination();
System.out.println("Destination "+dest);
String destTer= leg.get(l).getDestinationTerminal();
System.out.println("DestTer "+destTer);
String origin=leg.get(l).getOrigin();
System.out.println("origun "+origin);
String originTer=leg.get(l).getOriginTerminal();
System.out.println("OriginTer "+originTer);
int durationLeg= leg.get(l).getDuration();
System.out.println("durationleg "+durationLeg);
int mil= leg.get(l).getMileage();
System.out.println("Milleage "+mil);
}
}
}
//Pricing
List<PricingInfo> priceInfo= tripResults.get(i).getPricing();
for(int p=0; p<priceInfo.size(); p++){
String price= priceInfo.get(p).getSaleTotal();
System.out.println("Price "+price);
}
}
return;
} catch (IOException e) {
System.err.println(e.getMessage());
} catch (Throwable t) {
t.printStackTrace();
}
System.exit(1);
}
}

Related

Accessing TableRow columns in BigQuery Apache Beam

I am trying to
1.Read JSON events from Cloud Pub/Sub
2.Load the events from Cloud Pub/Sub to BigQuery every 15 minutes using file loads to save cost on streaming inserts.
3.The destination will differ based on "user_id" and "campaign_id" field in the JSON event, "user_id" will be dataset name and "campaign_id" will be the table name. The partition name comes from the event timestamp.
4.The schema for all tables stays same.
I am new to Java and Beam. I think my code mostly does what I am trying to do and I just a need little help here.
But I unable to access "campaign_id" and "user_id" field in the JSON message.
So, my events are not routing to the correct table.
package ...;
import com.google.api.services.bigquery.model.TableSchema;
import javafx.scene.control.TableRow;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO;
import org.apache.beam.sdk.io.gcp.bigquery.DynamicDestinations;
import org.apache.beam.sdk.io.gcp.bigquery.TableDestination;
import org.apache.beam.sdk.io.gcp.bigquery.TableRowJsonCoder;
import org.apache.beam.sdk.io.gcp.pubsub.PubsubIO;
import org.apache.beam.sdk.transforms.MapElements;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.SimpleFunction;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.ValueInSingleWindow;
import org.joda.time.Duration;
import org.joda.time.Instant;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED;
import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.Method.FILE_LOADS;
import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition.WRITE_APPEND;
public class ClickLogConsumer {
private static final int BATCH_INTERVAL_SECS = 15 * 60;
private static final String PROJECT = "pure-app";
public static PTransform<PCollection<String>, PCollection<com.google.api.services.bigquery.model.TableRow>> jsonToTableRow() {
return new JsonToTableRow();
}
private static class JsonToTableRow
extends PTransform<PCollection<String>, PCollection<com.google.api.services.bigquery.model.TableRow>> {
#Override
public PCollection<com.google.api.services.bigquery.model.TableRow> expand(PCollection<String> stringPCollection) {
return stringPCollection.apply("JsonToTableRow", MapElements.<String, com.google.api.services.bigquery.model.TableRow>via(
new SimpleFunction<String, com.google.api.services.bigquery.model.TableRow>() {
#Override
public com.google.api.services.bigquery.model.TableRow apply(String json) {
try {
InputStream inputStream = new ByteArrayInputStream(
json.getBytes(StandardCharsets.UTF_8.name()));
//OUTER is used here to prevent EOF exception
return TableRowJsonCoder.of().decode(inputStream, Coder.Context.OUTER);
} catch (IOException e) {
throw new RuntimeException("Unable to parse input", e);
}
}
}));
}
}
public static void main(String[] args) throws Exception {
Pipeline pipeline = Pipeline.create(options);
pipeline
.apply(PubsubIO.readStrings().withTimestampAttribute("timestamp").fromTopic("projects/pureapp-199410/topics/clicks"))
.apply(jsonToTableRow())
.apply("WriteToBQ",
BigQueryIO.writeTableRows()
.withMethod(FILE_LOADS)
.withWriteDisposition(WRITE_APPEND)
.withCreateDisposition(CREATE_IF_NEEDED)
.withTriggeringFrequency(Duration.standardSeconds(BATCH_INTERVAL_SECS))
.withoutValidation()
.to(new DynamicDestinations<TableRow, String>() {
#Override
public String getDestination(ValueInSingleWindow<TableRow> element) {
String tableName = "campaign_id"; // JSON message in Pub/Sub has "campaign_id" field, how do I access it here?
String datasetName = "user_id"; // JSON message in Pub/Sub has "user_id" field, how do I access it here?
Instant eventTimestamp = element.getTimestamp();
String partition = new SimpleDateFormat("yyyyMMdd").format(eventTimestamp);
return String.format("%s:%s.%s$%s", PROJECT, datasetName, tableName, partition);
}
#Override
public TableDestination getTable(String table) {
return new TableDestination(table, null);
}
#Override
public TableSchema getSchema(String destination) {
return getTableSchema();
}
}));
pipeline.run();
}
}
I arrived at the above code based on reading:
1.https://medium.com/myheritage-engineering/kafka-to-bigquery-load-a-guide-for-streaming-billions-of-daily-events-cbbf31f4b737
2.https://shinesolutions.com/2017/12/05/fun-with-serializable-functions-and-dynamic-destinations-in-cloud-dataflow/
3.https://beam.apache.org/documentation/sdks/javadoc/2.0.0/org/apache/beam/sdk/io/gcp/bigquery/DynamicDestinations.html
4.BigQueryIO - Write performance with streaming and FILE_LOADS
5.Inserting into BigQuery via load jobs (not streaming)
Update
import com.google.api.services.bigquery.model.TableFieldSchema;
import com.google.api.services.bigquery.model.TableRow;
import com.google.api.services.bigquery.model.TableSchema;
import com.google.api.services.bigquery.model.TimePartitioning;
import com.google.common.collect.ImmutableList;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO;
import org.apache.beam.sdk.io.gcp.bigquery.TableDestination;
import org.apache.beam.sdk.io.gcp.bigquery.TableRowJsonCoder;
import org.apache.beam.sdk.io.gcp.pubsub.PubsubIO;
import org.apache.beam.sdk.transforms.MapElements;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.SimpleFunction;
import org.apache.beam.sdk.values.PCollection;
import org.joda.time.Duration;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED;
import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.Method.FILE_LOADS;
import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition.WRITE_APPEND;
public class ClickLogConsumer {
private static final int BATCH_INTERVAL_SECS = 15 * 60;
private static final String PROJECT = "pure-app";
public static PTransform<PCollection<String>, PCollection<TableRow>> jsonToTableRow() {
return new JsonToTableRow();
}
private static class JsonToTableRow
extends PTransform<PCollection<String>, PCollection<TableRow>> {
#Override
public PCollection<TableRow> expand(PCollection<String> stringPCollection) {
return stringPCollection.apply("JsonToTableRow", MapElements.<String, com.google.api.services.bigquery.model.TableRow>via(
new SimpleFunction<String, TableRow>() {
#Override
public TableRow apply(String json) {
try {
InputStream inputStream = new ByteArrayInputStream(
json.getBytes(StandardCharsets.UTF_8.name()));
//OUTER is used here to prevent EOF exception
return TableRowJsonCoder.of().decode(inputStream, Coder.Context.OUTER);
} catch (IOException e) {
throw new RuntimeException("Unable to parse input", e);
}
}
}));
}
}
public static void main(String[] args) throws Exception {
Pipeline pipeline = Pipeline.create(options);
pipeline
.apply(PubsubIO.readStrings().withTimestampAttribute("timestamp").fromTopic("projects/pureapp-199410/topics/clicks"))
.apply(jsonToTableRow())
.apply(BigQueryIO.write()
.withTriggeringFrequency(Duration.standardSeconds(BATCH_INTERVAL_SECS))
.withMethod(FILE_LOADS)
.withWriteDisposition(WRITE_APPEND)
.withCreateDisposition(CREATE_IF_NEEDED)
.withSchema(new TableSchema().setFields(
ImmutableList.of(
new TableFieldSchema().setName("timestamp").setType("TIMESTAMP"),
new TableFieldSchema().setName("exchange").setType("STRING"))))
.to((row) -> {
String datasetName = row.getValue().get("user_id").toString();
String tableName = row.getValue().get("campaign_id").toString();
return new TableDestination(String.format("%s:%s.%s", PROJECT, datasetName, tableName), "Some destination");
})
.withTimePartitioning(new TimePartitioning().setField("timestamp")));
pipeline.run();
}
}
How about: String tableName = element.getValue().get("campaign_id").toString() and likewise for the dataset.
Besides, for inserting into time-partitioned tables, I strongly recommend using BigQuery's Column-Based Partitioning, instead of using a partition decorator in the table name. Please see "Loading historical data into time-partitioned BigQuery tables" in the javadoc - you'll need a timestamp column. (note that the javadoc has a typo: "time" vs "timestamp")

How do I configure spring-kafka to ignore messages in the wrong format?

We have an issue with one of our Kafka topics which is consumed by the DefaultKafkaConsumerFactory & ConcurrentMessageListenerContainer combination described here with a JsonDeserializer used by the Factory. Unfortunately someone got a little enthusiastic and published some invalid messages onto the topic. It appears that spring-kafka silently fails to process past the first of these messages. Is it possible to have spring-kafka log an error and continue? Looking at the error messages which are logged it seems that perhaps the Apache kafka-clients library should deal with the case that when iterating a batch of messages one or more of them may fail to parse?
The below code is an example test case illustrating this issue:
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.Serializer;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.StringSerializer;
import org.junit.ClassRule;
import org.junit.Test;
import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
import org.springframework.kafka.core.DefaultKafkaProducerFactory;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.listener.KafkaMessageListenerContainer;
import org.springframework.kafka.listener.MessageListener;
import org.springframework.kafka.listener.config.ContainerProperties;
import org.springframework.kafka.support.SendResult;
import org.springframework.kafka.support.serializer.JsonDeserializer;
import org.springframework.kafka.support.serializer.JsonSerializer;
import org.springframework.kafka.test.rule.KafkaEmbedded;
import org.springframework.kafka.test.utils.ContainerTestUtils;
import org.springframework.util.concurrent.ListenableFuture;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.springframework.kafka.test.hamcrest.KafkaMatchers.hasKey;
import static org.springframework.kafka.test.hamcrest.KafkaMatchers.hasValue;
/**
* #author jfreedman
*/
public class TestSpringKafka {
private static final String TOPIC1 = "spring.kafka.1.t";
#ClassRule
public static KafkaEmbedded embeddedKafka = new KafkaEmbedded(1, true, 1, TOPIC1);
#Test
public void submitMessageThenGarbageThenAnotherMessage() throws Exception {
final BlockingQueue<ConsumerRecord<String, JsonObject>> records = createListener(TOPIC1);
final KafkaTemplate<String, JsonObject> objectTemplate = createPublisher("json", new JsonSerializer<JsonObject>());
sendAndVerifyMessage(records, objectTemplate, "foo", new JsonObject("foo"), 0L);
// push some garbage text to Kafka which cannot be marshalled, this should not interrupt processing
final KafkaTemplate<String, String> garbageTemplate = createPublisher("garbage", new StringSerializer());
final SendResult<String, String> garbageResult = garbageTemplate.send(TOPIC1, "bar","bar").get(5, TimeUnit.SECONDS);
assertEquals(1L, garbageResult.getRecordMetadata().offset());
sendAndVerifyMessage(records, objectTemplate, "baz", new JsonObject("baz"), 2L);
}
private <T> KafkaTemplate<String, T> createPublisher(final String label, final Serializer<T> serializer) {
final Map<String, Object> producerProps = new HashMap<>();
producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, embeddedKafka.getBrokersAsString());
producerProps.put(ProducerConfig.CLIENT_ID_CONFIG, "TestPublisher-" + label);
producerProps.put(ProducerConfig.ACKS_CONFIG, "all");
producerProps.put(ProducerConfig.RETRIES_CONFIG, 2);
producerProps.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, 1);
producerProps.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 5000);
producerProps.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 5000);
producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, serializer.getClass());
final DefaultKafkaProducerFactory<String, T> pf = new DefaultKafkaProducerFactory<>(producerProps);
pf.setValueSerializer(serializer);
return new KafkaTemplate<>(pf);
}
private BlockingQueue<ConsumerRecord<String, JsonObject>> createListener(final String topic) throws Exception {
final Map<String, Object> consumerProps = new HashMap<>();
consumerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, embeddedKafka.getBrokersAsString());
consumerProps.put(ConsumerConfig.GROUP_ID_CONFIG, "TestConsumer");
consumerProps.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true);
consumerProps.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "100");
consumerProps.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, 15000);
consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, JsonDeserializer.class);
final DefaultKafkaConsumerFactory<String, JsonObject> cf = new DefaultKafkaConsumerFactory<>(consumerProps);
cf.setValueDeserializer(new JsonDeserializer<>(JsonObject.class));
final KafkaMessageListenerContainer<String, JsonObject> container = new KafkaMessageListenerContainer<>(cf, new ContainerProperties(topic));
final BlockingQueue<ConsumerRecord<String, JsonObject>> records = new LinkedBlockingQueue<>();
container.setupMessageListener((MessageListener<String, JsonObject>) records::add);
container.setBeanName("TestListener");
container.start();
ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic());
return records;
}
private void sendAndVerifyMessage(final BlockingQueue<ConsumerRecord<String, JsonObject>> records,
final KafkaTemplate<String, JsonObject> template,
final String key, final JsonObject value,
final long expectedOffset) throws InterruptedException, ExecutionException, TimeoutException {
final ListenableFuture<SendResult<String, JsonObject>> future = template.send(TOPIC1, key, value);
final ConsumerRecord<String, JsonObject> record = records.poll(5, TimeUnit.SECONDS);
assertThat(record, hasKey(key));
assertThat(record, hasValue(value));
assertEquals(expectedOffset, future.get(5, TimeUnit.SECONDS).getRecordMetadata().offset());
}
public static final class JsonObject {
private String value;
public JsonObject() {}
JsonObject(final String value) {
this.value = value;
}
public String getValue() {
return value;
}
public void setValue(final String value) {
this.value = value;
}
#Override
public boolean equals(final Object o) {
if (this == o) { return true; }
if (o == null || getClass() != o.getClass()) { return false; }
final JsonObject that = (JsonObject) o;
return Objects.equals(value, that.value);
}
#Override
public int hashCode() {
return Objects.hash(value);
}
#Override
public String toString() {
return "JsonObject{" +
"value='" + value + '\'' +
'}';
}
}
}
I have a solution but I don't know if it's the best one, I extended JsonDeserializer as follows which results in a null value being consumed by spring-kafka and requires the necessary downstream changes to handle that case.
class SafeJsonDeserializer[A >: Null](targetType: Class[A], objectMapper: ObjectMapper) extends JsonDeserializer[A](targetType, objectMapper) with Logging {
override def deserialize(topic: String, data: Array[Byte]): A = try {
super.deserialize(topic, data)
} catch {
case e: Exception =>
logger.error("Failed to deserialize data [%s] from topic [%s]".format(new String(data), topic), e)
null
}
}
Starting from the spring-kafka-2.x.x, we now have the comfort of declaring beans in the config file for the interface KafkaListenerErrorHandler with a implementation something as
#Bean
public ConsumerAwareListenerErrorHandler listen3ErrorHandler() {
return (m, e, c) -> {
this.listen3Exception = e;
MessageHeaders headers = m.getHeaders();
c.seek(new org.apache.kafka.common.TopicPartition(
headers.get(KafkaHeaders.RECEIVED_TOPIC, String.class),
headers.get(KafkaHeaders.RECEIVED_PARTITION_ID, Integer.class)),
headers.get(KafkaHeaders.OFFSET, Long.class));
return null;
};
}
more resources can be found at https://docs.spring.io/spring-kafka/reference/htmlsingle/#annotation-error-handling There is also another link with the similar issue: Spring Kafka error handling - v1.1.x and How to handle SerializationException after deserialization
Use ErrorHandlingDeserializer2. This is a delegating key/value deserializer that catches exceptions, returning them in the headers as serialized java objects.
Under consumer configuration, add/update the below lines:
import org.apache.kafka.clients.consumer.ConsumerConfig
import org.springframework.kafka.support.serializer.ErrorHandlingDeserializer2
configProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,
classOf[ErrorHandlingDeserializer2[JsonDeserializer]].getName)
configProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, classOf[ErrorHandlingDeserializer2[StringDeserializer]].getName)
configProps.put(ErrorHandlingDeserializer2.KEY_DESERIALIZER_CLASS, classOf[StringDeserializer].getName)
configProps.put(ErrorHandlingDeserializer2.VALUE_DESERIALIZER_CLASS, classOf[JsonDeserializer].getName)

Exception in thread "pool-1-thread-7" java.lang.NullPointerException

I'm trying to figure it out y i get this exception as i wrote in the title.
i'm building a multi thread MMU (memory management unit) project, first i initialize and add some pages into the ram and to my HD also and then i create processes and running all the system. a bit of information about the project:
into runConfig i;m reading a json file with a list of processCycles and every processCycles including a list of processCycle that including a list of pageIds with a list of data of those pages.
i think i did everything ok but still i get this exception, can somebody help me?
MMUDriver is the class that run all the systems:
package hit.driver;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import com.google.gson.Gson;
import com.google.gson.JsonIOException;
import com.google.gson.JsonSyntaxException;
import com.google.gson.stream.JsonReader;
import com.hit.algorithm.IAlgoCache;
import com.hit.algorithm.LRUAlgoCacheImpl;
import com.hit.algorithm.MRUAlgoCacheImpl;
import com.hit.algorithm.RandomReplacementAlgoCacheImpl;
import hit.memoryunits.HardDisk;
import hit.memoryunits.MemoryManagementUnit;
import hit.memoryunits.MemoryManagementUnitTest;
import hit.memoryunits.Page;
import hit.processes.ProcessCycles;
import hit.processes.Process;
import hit.processes.RunConfiguration;
#SuppressWarnings("unused")
public class MMUDriver {
private static final String CONFIG_FILE_NAME="Configuration.json";
// private static final String CONFIG_FILE_NAME="test.json";
//read from JSON file the lists of processCycles
private static RunConfiguration readConfigurationFile() throws JsonIOException, JsonSyntaxException, FileNotFoundException{
RunConfiguration runConfig = new Gson().fromJson(new JsonReader(new FileReader(CONFIG_FILE_NAME)), RunConfiguration.class);
return runConfig;
}
//
private static List<Process> createProcesses(List<ProcessCycles> processCycles,MemoryManagementUnit mmu ){
int i=0;
List<Process> processList = new ArrayList<Process>();
for ( ProcessCycles currentListProcess : processCycles) {
Process process = new Process(++i , mmu , currentListProcess);
processList.add(process);
}
return processList;
}
//
private static void runProcesses(List<Process> processes){
//Executor is a interface that take care of threads מתי מתחיל טרד מתי נגמר איך הם ירוצו במקביל וכו'
//thread pool - אוסף של threads
ExecutorService executorservice = Executors.newCachedThreadPool();
for (Process process : processes)
executorservice.execute(process);
executorservice.shutdown();//
}
//
public static void main(String[] args) throws java.lang.InterruptedException,
InvocationTargetException, JsonIOException, JsonSyntaxException, ClassNotFoundException, IOException{
CLI cli = new CLI(System.in,System.out);
String [] configuration;
while((configuration = cli.getConfiguration()) != null){
IAlgoCache<Long, Long> algo = null;
int capacity = Integer.valueOf(configuration[1]);
switch(configuration[0]){ //which configuration to play on LRU|MRU|RR
case "LRU":
algo = new LRUAlgoCacheImpl<Long, Long>(capacity);
break;
case "MRU":
algo = new MRUAlgoCacheImpl<Long, Long>(capacity);
break;
case "RR":
algo = new RandomReplacementAlgoCacheImpl<Long, Long>(capacity);
break;
}
MemoryManagementUnit mmu = new MemoryManagementUnit(algo, capacity);
MemoryManagementUnitTest.test(mmu);//add some pages to ram for test
RunConfiguration runConfig = readConfigurationFile(); //Getting the user request pages throw json file
List<ProcessCycles> processCycles = runConfig.getProcessesCycles();
List<Process> processes = createProcesses(processCycles , mmu);
runProcesses(processes);
Map<Long, Page<byte[]>> bla= mmu.getRam().getRamPages();
System.out.println("1");
// mmu.shoutDown();
}
}
}
Process is like a thread in this project
package hit.processes;
import java.io.IOException;
import java.util.List;
import hit.memoryunits.MemoryManagementUnit;
import hit.memoryunits.Page;
#SuppressWarnings("unused")
public class Process implements Runnable {
private int id;
private MemoryManagementUnit mmu;
private ProcessCycles processCycles;
private Thread processThread=null;
public Process(int id, MemoryManagementUnit mmu, ProcessCycles processCycles){
this.id=id;
this.mmu=mmu;
this.processCycles=processCycles;
}
public int getId(){
return id;
}
public void setId(int id){
this.id=id;
}
/*
public void start(){
if(processThread==null)
processThread = new Thread(this,"process"); //
processThread.start();
}*/
#Override
public void run() {
synchronized(mmu){
//runing on all the list of processCycles
for (ProcessCycle currentProcessCycle : processCycles.getProcessCycles()) {
List<Long> pages = currentProcessCycle.getPages();
List<byte[]> data = currentProcessCycle.getData();
Page<byte[]>[] pagesList = null;
//creating a boolean array to know if the page is for writing or reading in size of pages.size()
boolean [] writePages = new boolean[pages.size()];
for (int i = 0; i < pages.size(); i++)
if(data.get(i) == null) //if the page is empty
writePages[i] = false;
else{
writePages[i] = true ;//if there is data in the page and we want to write it to the ram
}
try {
// Getting the pages from ram to see which we shall update
pagesList = mmu.getPages(pages.toArray(new Long[pages.size()]), writePages);
}
catch (ClassNotFoundException e1) {
e1.printStackTrace();
} catch (IOException e1) {
e1.printStackTrace();
}
for (int j = 0; j < pages.size(); j++)
if(writePages[j] == true) //if the page isn't for read only
pagesList[j].setContent(data.get(j));
try {
Thread.sleep(currentProcessCycle.getSleepMs());
}
catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}
}
MMU class is that class that take care of all the requests from the user and manage all the pagging
package hit.memoryunits;
import java.io.FileNotFoundException;
import java.io.IOException;
import com.hit.algorithm.IAlgoCache;
public class MemoryManagementUnit {
private IAlgoCache<Long,Long> algo;
private RAM<Long, Page<byte[]>> ram;
public MemoryManagementUnit(IAlgoCache<Long, Long> algo, int ramCapacity) {
this.algo = algo;
this.ram= new RAM<Long, Page<byte[]>>(ramCapacity);
}
#SuppressWarnings("null")
public Page<byte[]> [] getPages(Long [] pageIds , boolean [] writePages) throws IOException, ClassNotFoundException{
for (int i = 0; i < pageIds.length; i++) {
if(algo.getElement((long) pageIds[i].hashCode())== null)//the page isn't exist in the cache
if(!ram.isFull())
{
if(writePages[i] == true){//if the page is for write we need to overwrite data
Page<byte[]> pageToRam = null;
pageToRam.setPageId(pageIds[i]);
pageToRam.setContent(null);
ram.addPage(pageToRam);// ram is not full need to upload from hd
}
else{//if the page is for read-only so we need to upload the page from HD
ram.addPage(HardDisk.getInstance().pageFault(pageIds[i]));// ram is not full need to upload from hd
}
algo.putElement(Long.valueOf(pageIds[i].hashCode()),pageIds[i]);//update cache
}
else
{
Page<byte[]> pageToRemove = ram.getPage(algo.putElement(Long.valueOf(pageIds[i].hashCode()),pageIds[i]));
ram.removePage(pageToRemove);
ram.addPage(HardDisk.getInstance().pageReplacement(pageToRemove,pageIds[i]));// ram full need to do replacement
algo.putElement(Long.valueOf(pageIds[i].hashCode()),pageIds[i]);//update cache
}
}
return ram.getPages(pageIds);
}
public RAM<Long, Page<byte[]>> getRam(){
return ram;
}
public IAlgoCache<Long,Long> getAlgo(){
return algo;
}
public void setAlgo(IAlgoCache<Long,Long> algo){
this.algo = algo;
}
public void setRam(RAM<Long, Page<byte[]>> ram){
this.ram = ram;
}
public void shoutDown() throws FileNotFoundException, IOException, ClassNotFoundException{
HardDisk.getInstance().saveToDisk(this.getRam().getRamPages());
}
}

Neo4j error caused by Lucene (Too many open files)

I've just started evaluating Neo4j to see how well its fits our use case.
I'm using the embedded Java API to insert edges and nodes into a graph.
After creating around 5000 nodes I get the following error (using Neo4j 2.1.6 and 2.1.7 on OS X Yosemite)
org.neo4j.graphdb.TransactionFailureException: Unable to commit transaction
Caused by: javax.transaction.xa.XAException
Caused by: org.neo4j.kernel.impl.nioneo.store.UnderlyingStorageException: java.io.FileNotFoundException: /Users/mihir.k/IdeaProjects/Turant/target/neo4j-hello-db/schema/label/lucene/_8zr.frq (Too many open files)
Caused by: java.io.FileNotFoundException: /Users/mihir.k/IdeaProjects/Turant/target/neo4j-hello-db/schema/label/lucene/_8zr.frq (Too many open files)
I've looked at numerous similar StackOverFlow questions and other related threads online. They all suggest increasing the max open files limit.
I've tried doing that.
These are my settings:
kern.maxfiles: 65536
kern.maxfilesperproc: 65536
However this hasn't fixed the error.
While the Neo4j code runs I tried using the lsof|wc -l command. The code always breaks when around 10000 files are open.
The following is the main class that deals with Neo4j:
import java.io.File;
import java.io.Serializable;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.neo4j.cypher.internal.compiler.v1_9.commands.True;
import org.neo4j.cypher.internal.compiler.v2_0.ast.False;
import org.neo4j.graphdb.*;
import org.neo4j.graphdb.factory.GraphDatabaseFactory;
import org.neo4j.graphdb.schema.Schema;
import org.neo4j.graphdb.schema.IndexDefinition;
import org.neo4j.graphdb.index.UniqueFactory;
import org.neo4j.graphdb.index.Index;
import org.neo4j.graphdb.index.IndexHits;
public class Neo4jDB implements Serializable {
private static final String DB_PATH = "target/neo4j-hello-db-spark";
IndexDefinition indexDefinition;
private static GraphDatabaseFactory dbFactory;
public static GraphDatabaseService db;
public void main(String[] args) {
System.out.println("Life is a disease, sexually transmitted and irrevocably fatal. Stop coding and read some Neil Gaiman.");
}
public void startDbInstance() {
db =new GraphDatabaseFactory().newEmbeddedDatabase(DB_PATH);
}
public Node createOrGetNode ( LabelsUser360 label , String key, String nodeName ,Map<String,Object> propertyMap)
{
System.out.println("Creating/Getting node");
try ( Transaction tx = db.beginTx() ) {
Node node;
if (db.findNodesByLabelAndProperty(label, key, nodeName).iterator().hasNext()) {
node = db.findNodesByLabelAndProperty(label, key, nodeName).iterator().next();
} else {
node = db.createNode(label);
node.setProperty(key, nodeName);
}
for (Map.Entry<String, Object> entry : propertyMap.entrySet()) {
node.setProperty(entry.getKey(), entry.getValue());
}
tx.success();
return node;
}
}
public void createUniquenessConstraint(LabelsUser360 label , String property)
{
try ( Transaction tx = db.beginTx() )
{
db.schema()
.constraintFor(label)
.assertPropertyIsUnique(property)
.create();
tx.success();
}
}
public void createOrUpdateRelationship(RelationshipsUser360 relationshipType ,Node startNode, Node endNode, Map<String,Object> propertyMap)
{
try ( Transaction tx = db.beginTx() ) {
if (startNode.hasRelationship(relationshipType, Direction.OUTGOING)) {
Relationship relationship = startNode.getSingleRelationship(relationshipType, Direction.OUTGOING);
for (Map.Entry<String, Object> entry : propertyMap.entrySet()) {
relationship.setProperty(entry.getKey(), entry.getValue());
}
} else {
Relationship relationship = startNode.createRelationshipTo(endNode, relationshipType);
for (Map.Entry<String, Object> entry : propertyMap.entrySet()) {
relationship.setProperty(entry.getKey(), entry.getValue());
}
}
tx.success();
}
}
public void registerShutdownHook( final GraphDatabaseService graphDb )
{
Runtime.getRuntime().addShutdownHook( new Thread()
{
#Override
public void run()
{
db.shutdown();
}
} );
}
}
There is another Neo4jAdapter class that is used to implement domain specific logic. It uses the Neo4jDB class to do add/update nodes/properties/relationships
import org.apache.lucene.index.IndexWriter;
import org.codehaus.jackson.map.ObjectMapper;
import org.json.*;
import org.neo4j.graphdb.*;
import org.neo4j.graphdb.factory.GraphDatabaseFactory;
import org.neo4j.graphdb.schema.IndexDefinition;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
public class Neo4jAdapter implements Serializable {
static Neo4jDB n4j = new Neo4jDB();
public static GraphDatabaseService db = Neo4jDB.db ;
public void begin() {
n4j.startDbInstance();
}
public static void main(String[] args) {}
public String graphPut(String jsonString) {
System.out.println("graphput called");
HashMap<String, Object> map = jsonToMap(jsonString); //Json deserializer
Node startNode = n4j.createOrGetNode(...);
Node endNode = n4j.createOrGetNode(...);
propertyMap = new HashMap<String, Object>();
propertyMap.put(....);
try (Transaction tx = Neo4jDB.db.beginTx()) {
Relationship relationship = startNode.getSingleRelationship(...);
if (relationship != null) {
Integer currentCount = (Integer) relationship.getProperty("count");
Integer updatedCount = currentCount + 1;
propertyMap.put("count", updatedCount);
} else {
Integer updatedCount = 1;
propertyMap.put("count", updatedCount);
}
tx.success();
}
n4j.createOrUpdateRelationship(RelationshipsUser360.BLAH, startNode, endNode, propertyMap);
}
}
}
return "Are you sponge worthy??";
}
}
Finally, there is a Sprak App that calls the "graphput" method of the Neo4jAdapter class. The relevant code snippet is (the following is scala+spark code) :
val graphdb : Neo4jAdapter = new Neo4jAdapter()
graphdb.begin()
linesEnriched.foreach(a=>graphdb.graphPutMap(a))
where 'a' is a json string and linesEnriched is a Spark RDD (basically a set of strings)

Convert Byte array to pdf for printing

I have a byte array and I need to send this byte array over the print server socket for printing as pdf. How do I convert this byte array into pdf bytes ?.
Basically, I created a pdf template and using PdfStamper, I am generating the pdf
PdfStamper stamper = new PdfStamper(pdfTemplate, out);
and then convering the ByteArrayOutputStream to byte array (out.toByteArray()). I am sending this byte array to another service which just picks the byte array and send it over the socket for printing. I tried to print this and this prints nothing but a blank page.
I guess, the printer is not recognizing this as a pdf. How do I tell printer that my byte array is pdf (How do I convert the byte array to pdf printable format)
calling this from method where I m passing my byte array and
DocFlavor flavor = DocFlavor.BYTE_ARRAY.AUTOSENSE;
You have to call this method by passing flavor and byte array Like this
new PrintTest().print(byteArray, flavor);
import java.awt.Graphics;
import java.awt.print.PageFormat;
import java.awt.print.Printable;
import java.awt.print.PrinterException;
import java.awt.print.PrinterJob;
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import java.util.Set;
import javax.print.Doc;
import javax.print.DocFlavor;
import javax.print.DocPrintJob;
import javax.print.PrintException;
import javax.print.PrintService;
import javax.print.SimpleDoc;
import javax.print.attribute.DocAttributeSet;
import javax.print.attribute.standard.PrinterStateReason;
import javax.print.attribute.standard.PrinterStateReasons;
import javax.print.attribute.standard.Severity;
import javax.print.event.PrintJobAttributeEvent;
import javax.print.event.PrintJobAttributeListener;
import javax.print.event.PrintJobEvent;
import javax.print.event.PrintJobListener;
import javax.print.event.PrintServiceAttributeEvent;
import javax.print.event.PrintServiceAttributeListener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class PrintTest implements PrintServiceAttributeListener,PrintJobListener,Doc, Printable, PrintJobAttributeListener {
protected Logger logger = LoggerFactory.getLogger(PrintTest.class);
public void print(final byte[] byteArray, final DocFlavor flavor) {
Thread newThread = new Thread(new Runnable() {
public void run() {
PrintService ps = PrinterJob.getPrinterJob().getPrintService();
ps.addPrintServiceAttributeListener(PrintTest.this);
DocPrintJob docJob = ps.createPrintJob();
docJob.addPrintJobAttributeListener(PrintTest.this, null);
docJob.addPrintJobListener(PrintTest.this);
Doc document = new SimpleDoc(byteArray, flavor, null);
try {
docJob.print(document,null);
}
catch (PrintException e) {
e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
}
}
});
newThread.start();
/**
PrintServiceAttributeSet attSet = ps.getAttributes();
PrinterStateReasons psr = ps.getAttribute(PrinterStateReasons.class);
if (psr != null) {
Set<PrinterStateReason> errors = psr.printerStateReasonSet(Severity.REPORT);
for (PrinterStateReason reason : errors)
System.out.printf(" Reason : %s",reason.getName());
logger.info();
} */
}
public void attributeUpdate(PrintServiceAttributeEvent psae) {
logger.info("attributeUpdate: "+psae.getAttributes());
}
public void printDataTransferCompleted(PrintJobEvent pje) {
logger.info("Transfer completed");
}
public void printJobCompleted(PrintJobEvent pje) {
logger.info("Completed");
}
public void printJobFailed(PrintJobEvent pje) {
logger.info("Failed");
PrinterStateReasons psr = pje.getPrintJob().getPrintService().getAttribute(PrinterStateReasons.class);
if (psr != null) {
Set<PrinterStateReason> errors = psr.printerStateReasonSet(Severity.REPORT);
for (PrinterStateReason reason : errors)
logger.info(" Reason : %s",reason.getName());
logger.info("\n");
}
}
public void printJobCanceled(PrintJobEvent pje) {
logger.info("Canceled");
}
public void printJobNoMoreEvents(PrintJobEvent pje) {
logger.info("No more events");
logger.info("printJobNoMoreEvents: "+pje.getPrintEventType());
}
public void printJobRequiresAttention(PrintJobEvent pje) {
logger.info("Job requires attention");
PrinterStateReasons psr = pje.getPrintJob().getPrintService().getAttribute(PrinterStateReasons.class);
if (psr != null) {
Set<PrinterStateReason> errors = psr.printerStateReasonSet(Severity.REPORT);
for (PrinterStateReason reason : errors)
logger.info(" Reason : %s",reason.getName());
logger.info("\n");
}
}
public DocFlavor getDocFlavor() {
return DocFlavor.SERVICE_FORMATTED.PRINTABLE; //To change body of implemented methods use File | Settings | File Templates.
}
public Object getPrintData() throws IOException {
return this;
}
public DocAttributeSet getAttributes() {
return null; //To change body of implemented methods use File | Settings | File Templates.
}
public Reader getReaderForText() throws IOException {
return null; //To change body of implemented methods use File | Settings | File Templates.
}
public InputStream getStreamForBytes() throws IOException {
return null; //To change body of implemented methods use File | Settings | File Templates.
}
public int print(Graphics graphics, PageFormat pageFormat, int pageIndex) throws PrinterException {
return pageIndex == 0 ? PAGE_EXISTS : NO_SUCH_PAGE; //To change body of implemented methods use File | Settings | File Templates.
}
public void attributeUpdate(PrintJobAttributeEvent pjae) {
logger.info("Look out");
}
}