I've recently been learning about how to use the Kafka Streams client and one thing that I've been struggling with is how to switch from the default state store (RocksDB) to a custom state store using something like Redis. The Confluent documentation makes it clear you have to implement the StateStore interface for your custom store and you must provide an implementation of StoreBuilder for creating instances of that store.
Here's what I have so far for my custom store. I've also added a simple write method to append a new entry into a specified stream via the Redis XADD command.
public class MyCustomStore<K,V> implements StateStore, MyWriteableCustomStore<K,V> {
private String name;
private volatile boolean open = false;
private boolean loggingEnabled = false;
public MyCustomStore(String name, boolean loggingEnabled) {
this.name = name;
this.loggingEnabled = loggingEnabled;
}
#Override
public String name() {
return this.name;
}
#Override
public void init(ProcessorContext context, StateStore root) {
if (root != null) {
// register the store
context.register(root, (key, value) -> {
write(key.toString(), value.toString());
});
}
this.open = true;
}
#Override
public void flush() {
// TODO Auto-generated method stub
}
#Override
public void close() {
// TODO Auto-generated method stub
}
#Override
public boolean persistent() {
// TODO Auto-generated method stub
return true;
}
#Override
public boolean isOpen() {
// TODO Auto-generated method stub
return false;
}
#Override
public void write(String key, String value) {
try(Jedis jedis = new Jedis("localhost", 6379)) {
Map<String, String> hash = new HashMap<>();
hash.put(key, value);
jedis.xadd("MyStream", StreamEntryID.NEW_ENTRY, hash);
}
}
}
public class MyCustomStoreBuilder implements StoreBuilder<MyCustomStore<String,String>> {
private boolean cached = true;
private String name;
private Map<String,String> logConfig=new HashMap<>();
private boolean loggingEnabled;
public MyCustomStoreBuilder(String name, boolean loggingEnabled){
this.name = name;
this.loggingEnabled = loggingEnabled;
}
#Override
public StoreBuilder<MyCustomStore<String,String>> withCachingEnabled() {
this.cached = true;
return this;
}
#Override
public StoreBuilder<MyCustomStore<String,String>> withCachingDisabled() {
this.cached = false;
return null;
}
#Override
public StoreBuilder<MyCustomStore<String,String>> withLoggingEnabled(Map<String, String> config) {
loggingEnabled=true;
return this;
}
#Override
public StoreBuilder<MyCustomStore<String,String>> withLoggingDisabled() {
this.loggingEnabled = false;
return this;
}
#Override
public MyCustomStore<String,String> build() {
return new MyCustomStore<String,String>(this.name, this.loggingEnabled);
}
#Override
public Map<String, String> logConfig() {
return logConfig;
}
#Override
public boolean loggingEnabled() {
return loggingEnabled;
}
#Override
public String name() {
return name;
}
}
And here's what my setup and topology look like.
#Bean
public KafkaStreams kafkaStreams(KafkaProperties kafkaProperties) {
final Properties props = new Properties();
props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaProperties.getBootstrapServers());
props.put(StreamsConfig.APPLICATION_ID_CONFIG, appName);
props.put(AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, schemaRegistryUrl);
props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Long().getClass());
props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Double().getClass());
props.put(StreamsConfig.STATE_DIR_CONFIG, "data");
props.put(StreamsConfig.APPLICATION_SERVER_CONFIG, appServerConfig);
props.put(JsonDeserializer.VALUE_DEFAULT_TYPE, JsonNode.class);
props.put(DEFAULT_DESERIALIZATION_EXCEPTION_HANDLER_CLASS_CONFIG, LogAndContinueExceptionHandler.class);
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
final String storeName = "the-custome-store";
Topology topology = new Topology();
// Create CustomStoreSupplier for store name the-custom-store
MyCustomStoreBuilder customStoreBuilder = new MyCustomStoreBuilder(storeName, false);
topology.addSource("input","inputTopic");
topology.addProcessor("redis-processor", () -> new RedisProcessor(storeName), "input");
topology.addStateStore(customStoreBuilder, "redis-processor");
KafkaStreams kafkaStreams = new KafkaStreams(topology, props);
kafkaStreams.start();
return kafkaStreams;
}
public class MyCustomStoreType<K,V> implements QueryableStoreType<MyReadableCustomStore<String,String>> {
#Override
public boolean accepts(StateStore stateStore) {
return stateStore instanceof MyCustomStore;
}
#Override
public MyReadableCustomStore<String,String> create(final StateStoreProvider storeProvider, final String storeName) {
return new MyCustomStoreTypeWrapper<>(storeProvider, storeName, this);
}
}
public class MyCustomStoreTypeWrapper<K,V> implements MyReadableCustomStore<K,V> {
private final QueryableStoreType<MyReadableCustomStore<String, String>> customStoreType;
private final String storeName;
private final StateStoreProvider provider;
public MyCustomStoreTypeWrapper(final StateStoreProvider provider,
final String storeName,
final QueryableStoreType<MyReadableCustomStore<String, String>> customStoreType) {
this.provider = provider;
this.storeName = storeName;
this.customStoreType = customStoreType;
}
#Override
public String read(String key) {
try (Jedis jedis = new Jedis("localhost", 6379)) {
StreamEntryID start = new StreamEntryID(0, 0);
StreamEntryID end = null; // null -> until the last item in the stream
int count = 2;
List<StreamEntry> list = jedis.xrange("MyStream", start, end, count);
if (list != null) {
// Get the most recently added item, which is also the last item
StreamEntry streamData = list.get(list.size() - 1);
return streamData.toString();
} else {
System.out.println("No new data in the stream");
}
return "";
}
}
}
// This throws the InvalidStateStoreException when I try to get access to the custom store
MyReadableCustomStore<String,String> store = streams.store("the-custome-store", new MyCustomStoreType<String,String>());
String value = store.read("testKey");
So, my question is how do I actually get the state store data to persist into Redis now? I feel like I'm missing something in the state store initialization or with the StateRestoreCallback. Any help or clarification with this would be greatly appreciated.
It looks to me that you have the store wired up to the topology correctly. But you don't have any processors using the store.
It could look something like this:
final String storeName = "the-custome-store";
MyCustomStoreBuilder customStoreBuilder = new MyCustomStoreBuilder(storeName, false);
Topology topology = new Topology()
topology.addSource("input", "input-topic");
// makes the processor a child of the source node
// the source node forwards its records to the child processor node
topology.addProcessor("redis-processor", () -> new RedisProcessor(storeName), "input");
// add the store and specify the processor(s) that access the store
topology.addStateStore(storeBuilder, "redis-processor");
class RedisProcessor implements Processor<byte[], byte[]> {
final String storeName;
MyCustomStore<byte[],byte[]> stateStore;
public RedisProcessor(String storeName) {
this.storeName = storeName;
}
#Override
public void init(ProcessorContext context) {
stateStore = (MyCustomeStore<byte[], byte[]>) context.getStateStore(storeName);
}
#Override
public void process(byte[] key, byte[] value) {
stateStore.write(key, value);
}
#Override
public void close() {
}
}
HTH, and let me know how it works out for you.
Update to answer from comments:
I think you need to update MyCustomStore.isOpen() to return the open variable.
Right now it's hardcoded to return false
Override
public boolean isOpen() {
// TODO Auto-generated method stub
return false;
}
I created a table-viewer with two columns in it.first column name is "Fristname" and second column name "lastname". I added editor support to both the columns but i able to do edit/select only in the first column. In my second column not able to do editing/selecting. Don't know why some one please help me? Following is the code snippet.
public class ViewPart1 extends ViewPart {
public ViewPart1() {
// TODO Auto-generated constructor stub
}
private ResourceManager resourceManager = new LocalResourceManager(
JFaceResources.getResources());
#Override
public void createPartControl(Composite parent) {
// re-use an existing image
final Image image = FieldDecorationRegistry.getDefault()
.getFieldDecoration(FieldDecorationRegistry.DEC_INFORMATION)
.getImage();
TableViewer tblView = new TableViewer(parent);
final Table table = tblView.getTable();
table.setHeaderVisible(true);
table.setLinesVisible(true);
TableViewerColumn fn = new TableViewerColumn(tblView, SWT.BORDER, 0);
fn.getColumn().setWidth(150);
fn.getColumn().setText("Firstname");
fn.setLabelProvider(new ColumnLabelProvider() {
#Override
public String getText(Object element) {
Person p = (Person) element;
return p.getFirstName();
}
#Override
public Image getImage(Object element) {
return image;
}
});
// fn.setEditingSupport(new EditColumn(tblView));
fn = new TableViewerColumn(tblView, SWT.BORDER, 1);
fn.getColumn().setWidth(150);
fn.getColumn().setText("Last name");
fn.setLabelProvider(new ColumnLabelProvider() {
#Override
public String getText(Object element) {
Person p = (Person) element;
return p.getLastNAme();
}
#Override
public Image getImage(Object element) {
return image;
}
});
// fn.setEditingSupport(new EditColumn(tblView));
tblView.setContentProvider(new QContentProvider());
ArrayList<Person> list = new ArrayList<Person>();
list.add(new Person("a", "b"));
list.add(new Person("C", "D"));
tblView.setInput(list);
tblView.refresh();
}
#Override
public void setFocus() {
// TODO Auto-generated method stub
}
}
Below EditColumn class Code
public class EditColumn extends EditingSupport {
private final TableViewer viewer;
private final CellEditor editor;
public EditColumn(TableViewer viewer) {
super(viewer);
this.viewer = viewer;
this.editor = new TextCellEditor(viewer.getTable());
}
#Override
protected boolean canEdit(Object element) {
System.out.println("can edit");
return true;
}
#Override
protected CellEditor getCellEditor(Object element) {
return editor;
}
#Override
protected Object getValue(Object element) {
// TODO Auto-generated method stub
return ((Person) element).getFirstName();
}
#Override
protected void setValue(Object element, Object value) {
((Person) element).setFirstName(String.valueOf(value));
viewer.update(element, null);
}
}
Your EditColumn class is always using the getFirstName and setFirstName methods of Person so although you can edit the last name column you are not using the correct values.
You need to use different EditingSupport classes for each column.
I added rerun test by Testng, but have a problem with duplicate tests in test report. Could you help me with this
private int retryCount = 0;
private int maxRetryCount = 1;
#Override
public boolean retry(ITestResult result) {
if (retryCount < maxRetryCount) {
System.out.println("Retrying test " + result.getName() + " with status "
+ getResultStatusName(result.getStatus()) + " for the " + (retryCount+1) + " time(s).");
retryCount++;
return true;
}
return false;
}
public String getResultStatusName(int status) {
String resultName = null;
if(status==1)
resultName = "SUCCESS";
if(status==2)
resultName = "FAILURE";
if(status==3)
resultName = "SKIP";
return resultName;
}
And
public class RetryListener implements IAnnotationTransformer {
#Override
public void transform(ITestAnnotation testannotation, Class testClass,
Constructor testConstructor, Method testMethod) {
IRetryAnalyzer retry = testannotation.getRetryAnalyzer();
if (retry == null) {
testannotation.setRetryAnalyzer(iOpiumListener.class);
}
}
}
But in test report displayed or two failed or one passed and one failed test
Olga, I believe that you're missing additional listener that will clean your test results:
public class TestListener implements ITestListener {
#Override
public void onTestStart(ITestResult result) {
}
#Override
public void onTestSuccess(ITestResult result) {
}
#Override
public void onTestFailure(ITestResult result) {
}
#Override
public void onTestSkipped(ITestResult result) {
}
#Override
public void onTestFailedButWithinSuccessPercentage(ITestResult result) {
}
#Override
public void onStart(ITestContext context) {
}
#Override
public void onFinish(ITestContext context) {
Set<ITestResult> failedTests = context.getFailedTests().getAllResults();
for (ITestResult temp : failedTests) {
ITestNGMethod method = temp.getMethod();
if (context.getFailedTests().getResults(method).size() > 1) {
failedTests.remove(temp);
} else {
if (context.getPassedTests().getResults(method).size() > 0) {
failedTests.remove(temp);
}
}
}
}
}
However, please, be aware of the fact that it'll work only with pure TestNG, case you have additional tools that rely on TestNG, i.e. Cucumber - It'll be necessary to clean those reports separately.
You can catch results which are incorrect (onTestFailure) and delete them when all the tests are done (onFinish)
public class RetryAnalyzer implements IRetryAnalyzer {
private static int retryCount = 0;
private static final int maxRetryCount = 1;
private static boolean isRerun = false;
#Override
public boolean retry(ITestResult result) {
if (retryCount < maxRetryCount) {
retryCount++;
isRerun = true;
return true;
} else {
retryCount = 0;
isRerun = false;
}
return false;
}
public boolean isRerun() {
return isRerun;
}
private String getResultStatusName(int status) {
String resultName = null;
if (status == 1)
resultName = "SUCCESS";
if (status == 2)
resultName = "FAILURE";
if (status == 3)
resultName = "SKIP";
return resultName;
}
}
public class TestListener implements ITestListener, IAnnotationTransformer {
private List<ITestResult> failedTestsToRemove = new ArrayList<>();
#Override
public void onTestStart(ITestResult result) {
}
#Override
public void onTestSuccess(ITestResult result) {
}
#Override
public void onTestFailure(ITestResult result) {
RetryAnalyzer retryAnalyzer = (RetryAnalyzer) result.getMethod().getRetryAnalyzer();
if (retryAnalyzer.isRerun()) {
failedTestsToRemove.add(result);
}
}
#Override
public void onTestSkipped(ITestResult result) {
}
#Override
public void onTestFailedButWithinSuccessPercentage(ITestResult result) {
}
#Override
public void onStart(ITestContext context) {
}
#Override
public void onFinish(ITestContext context) {
for (ITestResult result : failedTestsToRemove) {
context.getFailedTests().removeResult(result);
}
}
#Override
public void transform(ITestAnnotation testannotation, Class testClass, Constructor testConstructor,
Method testMethod) {
IRetryAnalyzer retryAnalyzer = testannotation.getRetryAnalyzer();
if (retryAnalyzer == null) {
testannotation.setRetryAnalyzer(RetryAnalyzer.class);
}
}
}
So for a project I have to create a LinkedQueue class but without a counter variable so I can't exactly keep track of how many elements are in the queue.
I need to create a size method to return the number of elements in the queue but I have no idea how to do it...here's my code:
package animal;
import exceptions.EmptyQueueException;
/**
* #author Sharon Umute
* Comp 139 001B
*/
public class LinkedQueue<T> implements QueueADT<T>{
SinglyLinkedNode<T> tail,head;
public LinkedQueue(){
head=tail=null;
}
#Override
public void enqueue(T element) {
SinglyLinkedNode<T> node =new SinglyLinkedNode<T>(element);
if(isEmpty()){
head = node;
}else{
tail.setNext(node);
}
tail=node;
}
#Override
public T dequeue() throws EmptyQueueException {
if (isEmpty()){
throw new EmptyQueueException("queue");
}else{
T result = head.getElement();
head=head.getNext();
if(isEmpty()){
tail=null;
}
return result;
}
}
#Override
public T first() throws EmptyQueueException {
if (isEmpty()){
throw new EmptyQueueException("queue");
}else{
T result=head.getElement();
return result;
}
}
#Override
public boolean isEmpty() {
return(head.getElement()==null);
}
#Override
public int size() {
}
}
I think this is should be a good way to implement size method:
Iterator it = this.head.iterator();
int i = 0;
while (it.hasNext())
{
LinkedQueue node = it.Next();
i++;
}
return i;
I found a tutorial and it looks like this:
package com.djrobotfreak.SVTest;
public class Tutorial2D extends Activity {
#Override
public void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
requestWindowFeature(Window.FEATURE_NO_TITLE);
setContentView(new Panel(this));
}
class Panel extends SurfaceView implements SurfaceHolder.Callback {
private TutorialThread _thread;
public Panel(Context context) {
super(context);
getHolder().addCallback(this);
_thread = new TutorialThread(getHolder(), this);
}
#Override
public void onDraw(Canvas canvas) {
Bitmap _scratch = BitmapFactory.decodeResource(getResources(), R.drawable.ic_launcher);
canvas.drawColor(Color.BLACK);
canvas.drawBitmap(_scratch, 10, 10, null);
}
#Override
public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
// TODO Auto-generated method stub
}
#Override
public void surfaceCreated(SurfaceHolder holder) {
_thread.setRunning(true);
_thread.start();
}
#Override
public void surfaceDestroyed(SurfaceHolder holder) {
// simply copied from sample application LunarLander:
// we have to tell thread to shut down & wait for it to finish, or else
// it might touch the Surface after we return and explode
boolean retry = true;
_thread.setRunning(false);
while (retry) {
try {
_thread.join();
retry = false;
} catch (InterruptedException e) {
// we will try it again and again...
}
}
}
}
class TutorialThread extends Thread {
private SurfaceHolder _surfaceHolder;
private Panel _panel;
private boolean _run = false;
public TutorialThread(SurfaceHolder surfaceHolder, Panel panel) {
_surfaceHolder = surfaceHolder;
_panel = panel;
}
public void setRunning(boolean run) {
_run = run;
}
#Override
public void run() {
Canvas c;
while (_run) {
c = null;
try {
c = _surfaceHolder.lockCanvas(null);
synchronized (_surfaceHolder) {
_panel.onDraw(c);
}
} finally {
// do this in a finally so that if an exception is thrown
// during the above, we don't leave the Surface in an
// inconsistent state
if (c != null) {
_surfaceHolder.unlockCanvasAndPost(c);
}
}
}
}
}
}
and it does not work, no matter what I do. I am trying to convert my code to surfaceview but I cant find any surfaceview programs that even work (besides the android-provided ones). Does anyone know what the error even is saying?
Here is my logcat info: http://shrib.com/oJB5Bxqs
If you get a ClassNotFoundException, you should check the Manifest file.
Click on the Application tab and look on the botton right side under "Attributes for".
If there is a red X mark under your Class Name, then click on the "Name" link and locate the correct class to load.