Concurrently modified some keys - ignite

I am using Ignite 2.13.x for one of my requirement, I need to maintain cache in Ignite cluster and some of the keys value will be updates concurrently and selection also happens parallel. So here i am facing issue like,
java.sql.SQLException: Failed to update some keys because they had been modified concurrently [keys=[ROWKEY [idHash=628930863, hash=1637498765, rowkey=6, id=765436]]]
at org.apache.ignite.internal.jdbc.thin.JdbcThinConnection.sendRequest(JdbcThinConnection.java:1009)
at org.apache.ignite.internal.jdbc.thin.JdbcThinStatement.execute0(JdbcThinStatement.java:234)
at org.apache.ignite.internal.jdbc.thin.JdbcThinPreparedStatement.executeWithArguments(JdbcThinPreparedStatement.java:252)
at org.apache.ignite.internal.jdbc.thin.JdbcThinPreparedStatement.executeUpdate(JdbcThinPreparedStatement.java:96)
at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.tomcat.jdbc.pool.StatementFacade$StatementProxy.invoke(StatementFacade.java:118)
at com.sun.proxy.$Proxy16.executeUpdate(Unknown Source)
This was the configuration using at server side,
<bean class="org.apache.ignite.configuration.IgniteConfiguration"
id="igniteConfiguration">
<property name="cacheKeyConfiguration">
<list>
<bean class="org.apache.ignite.cache.CacheKeyConfiguration">
<property name="typeName" value="ROWKEY" />
<property name="affinityKeyFieldName" value="rowkey" />
</bean>
</list>
</property>
<property name="clientMode" value="false" />
<property name="gridName" value="6d-ignite-grid"></property>
<property name="cacheConfiguration">
<list>
<bean class="org.apache.ignite.configuration.CacheConfiguration">
<property name="name" value="persons"></property>
<!-- Set cache mode. -->
<property name="cacheMode" value="PARTITIONED" />
<!-- Set number of backups to 0 -->
<property name="backups" value="0" />
<property name="onheapCacheEnabled" value="false" />
<property name="statisticsEnabled" value="true" />
<property name="memoryPolicyName" value="Default_Region" />
<property name="copyOnRead" value="false" />
<property name="writeSynchronizationMode" value="FULL_SYNC" />
<property name="affinity">
<bean
class="org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction">
<property name="excludeNeighbors" value="true" />
<property name="partitions" value="10" />
<property name="affinityBackupFilter">
<bean
class="org.apache.ignite.cache.affinity.rendezvous.ClusterNodeAttributeAffinityBackupFilter">
<constructor-arg>
<array value-type="java.lang.String">
<value>AVAILABILITY_ZONE</value>
</array>
</constructor-arg>
</bean>
</property>
</bean>
</property>
<property name="queryEntities">
<list>
<bean class="org.apache.ignite.cache.QueryEntity">
<property name="keyType" value="ROWKEY"></property>
<property name="valueType" value="PERSONS"></property>
<property name="fields">
<map>
<entry key="rowkey" value="java.lang.Long"></entry>
<entry key="id" value="java.lang.Long"></entry>
<entry key="name" value="java.lang.String"></entry>
<entry key="birthday" value="java.lang.String"></entry>
<entry key="gender" value="java.lang.String"></entry>
<entry key="amount" value="java.lang.Long"></entry>
<entry key="salary" value="java.lang.Long"></entry>
<entry key="orgid" value="java.lang.String"></entry>
</map>
</property>
<property name="keyFields">
<set>
<value>rowkey</value>
<value>id</value>
</set>
</property>
</bean>
</list>
</property>
</bean>
</list>
</property>
<property name="includeEventTypes">
<list>
<!--Task execution events -->
<util:constant
static-field="org.apache.ignite.events.EventType.EVT_TASK_STARTED" />
<util:constant
static-field="org.apache.ignite.events.EventType.EVT_TASK_FINISHED" />
<util:constant
static-field="org.apache.ignite.events.EventType.EVT_TASK_FAILED" />
<util:constant
static-field="org.apache.ignite.events.EventType.EVT_TASK_TIMEDOUT" />
<util:constant
static-field="org.apache.ignite.events.EventType.EVT_TASK_SESSION_ATTR_SET" />
<util:constant
static-field="org.apache.ignite.events.EventType.EVT_TASK_REDUCED" />
<!--Cache events -->
<util:constant
static-field="org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_PUT" />
<util:constant
static-field="org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_READ" />
<util:constant
static-field="org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_REMOVED" />
</list>
</property>
<property name="peerClassLoadingEnabled" value="true"></property>
<property name="discoverySpi">
<bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
<property name="ipFinder">
<bean
class="org.apache.ignite.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinder">
<property name="addresses">
<list>
<!-- In distributed environment, replace with actual host IP address. -->
<value>127.0.0.1:47500..47509</value>
</list>
</property>
</bean>
</property>
</bean>
</property>
<property name="memoryConfiguration">
<bean class="org.apache.ignite.configuration.MemoryConfiguration">
<property name="defaultMemoryPolicyName" value="Default_Region" />
<property name="pageSize" value="4096" />
<property name="systemCacheInitialSize" value="#{40 * 1024 * 1024}" />
<property name="systemCacheMaxSize" value="#{40 * 1024 * 1024}" />
<property name="memoryPolicies">
<list>
<bean
class="org.apache.ignite.configuration.MemoryPolicyConfiguration">
<property name="name" value="Default_Region" />
<property name="initialSize" value="#{20 * 1024 * 1024}" />
<property name="maxSize" value="#{20 * 1024 * 1024}" />
<property name="pageEvictionMode" value="RANDOM_2_LRU" />
<property name="evictionThreshold" value="0.6" />
<property name="metricsEnabled" value="true" />
</bean>
</list>
</property>
</bean>
</property>
</bean>
Thanks for help
Ajay Babu Maguluri.

The error message indicates that the value was updated in parallel during the operation execution and the failed operation should be retried. It's expected to get such a type of error in the case of concurrent atomic updates for the same key in the cache. Or you can use transactions as an alternative way to avoid that.
Also, you can find more details regarding cache atomicity modes here:
https://ignite.apache.org/docs/latest/configuring-caches/atomicity-modes#atomicity-modes

Related

Enable SSL in ActiveMQ - connection getting timed out while accessing web console over https

I have a RHEL 7 server where I have ActiveMQ installed (version : apache-activemq-5.15.11).I am trying to run the web console over https and hence I went through the instructions mentioned in the documentation here and made changes in my jetty.xml and un-commented the following section:
<!--
Enable this connector if you wish to use https with web console
-->
<!-- bean id="SecureConnector" class="org.eclipse.jetty.server.ServerConnector">
<constructor-arg ref="Server" />
<constructor-arg>
<bean id="handlers" class="org.eclipse.jetty.util.ssl.SslContextFactory">
<property name="keyStorePath" value="${activemq.conf}/broker.ks" />
<property name="keyStorePassword" value="password" />
</bean>
</constructor-arg>
<property name="port" value="8162" />
</bean -->
when I am trying to access the web console like the following:
https://myserver.com:8162/admin/index.jsp
I am getting connection getting timed out in the browser.
Questions :
Am I trying to access it properly - via port 8162? It works fine with port 8161 and over http like the following:
http://myserver.com:8161/admin/index.jsp
Are there any additional changes needed to make it work over https?
My jetty.xml is below for reference:
<!--
Licensed to the Apache Software Foundation (ASF) under one or more contributor
license agreements. See the NOTICE file distributed with this work for additional
information regarding copyright ownership. The ASF licenses this file to You under
the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or
agreed to in writing, software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied. See the License for the specific language governing permissions and
limitations under the License.
-->
<!--
An embedded servlet engine for serving up the Admin consoles, REST and Ajax APIs and
some demos Include this file in your configuration to enable ActiveMQ web components
e.g. <import resource="jetty.xml"/>
-->
<beans xmlns="http://www.springframework.org/schema/beans" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
<bean id="securityLoginService" class="org.eclipse.jetty.security.HashLoginService">
<property name="name" value="ActiveMQRealm" />
<property name="config" value="${activemq.conf}/jetty-realm.properties" />
</bean>
<bean id="securityConstraint" class="org.eclipse.jetty.util.security.Constraint">
<property name="name" value="BASIC" />
<property name="roles" value="user,admin" />
<!-- set authenticate=false to disable login -->
<property name="authenticate" value="true" />
</bean>
<bean id="adminSecurityConstraint" class="org.eclipse.jetty.util.security.Constraint">
<property name="name" value="BASIC" />
<property name="roles" value="admin" />
<!-- set authenticate=false to disable login -->
<property name="authenticate" value="true" />
</bean>
<bean id="securityConstraintMapping" class="org.eclipse.jetty.security.ConstraintMapping">
<property name="constraint" ref="securityConstraint" />
<property name="pathSpec" value="/api/*,/admin/*,*.jsp" />
</bean>
<bean id="adminSecurityConstraintMapping" class="org.eclipse.jetty.security.ConstraintMapping">
<property name="constraint" ref="adminSecurityConstraint" />
<property name="pathSpec" value="*.action" />
</bean>
<bean id="rewriteHandler" class="org.eclipse.jetty.rewrite.handler.RewriteHandler">
<property name="rules">
<list>
<bean id="header" class="org.eclipse.jetty.rewrite.handler.HeaderPatternRule">
<property name="pattern" value="*"/>
<property name="name" value="X-FRAME-OPTIONS"/>
<property name="value" value="SAMEORIGIN"/>
</bean>
<bean id="header" class="org.eclipse.jetty.rewrite.handler.HeaderPatternRule">
<property name="pattern" value="*"/>
<property name="name" value="X-XSS-Protection"/>
<property name="value" value="1; mode=block"/>
</bean>
<bean id="header" class="org.eclipse.jetty.rewrite.handler.HeaderPatternRule">
<property name="pattern" value="*"/>
<property name="name" value="X-Content-Type-Options"/>
<property name="value" value="nosniff"/>
</bean>
</list>
</property>
</bean>
<bean id="secHandlerCollection" class="org.eclipse.jetty.server.handler.HandlerCollection">
<property name="handlers">
<list>
<ref bean="rewriteHandler"/>
<bean class="org.eclipse.jetty.webapp.WebAppContext">
<property name="contextPath" value="/admin" />
<property name="resourceBase" value="${activemq.home}/webapps/admin" />
<property name="logUrlOnStart" value="true" />
</bean>
<bean class="org.eclipse.jetty.webapp.WebAppContext">
<property name="contextPath" value="/api" />
<property name="resourceBase" value="${activemq.home}/webapps/api" />
<property name="logUrlOnStart" value="true" />
</bean>
<bean class="org.eclipse.jetty.server.handler.ResourceHandler">
<property name="directoriesListed" value="false" />
<property name="welcomeFiles">
<list>
<value>index.html</value>
</list>
</property>
<property name="resourceBase" value="${activemq.home}/webapps/" />
</bean>
<bean id="defaultHandler" class="org.eclipse.jetty.server.handler.DefaultHandler">
<property name="serveIcon" value="false" />
</bean>
</list>
</property>
</bean>
<bean id="securityHandler" class="org.eclipse.jetty.security.ConstraintSecurityHandler">
<property name="loginService" ref="securityLoginService" />
<property name="authenticator">
<bean class="org.eclipse.jetty.security.authentication.BasicAuthenticator" />
</property>
<property name="constraintMappings">
<list>
<ref bean="adminSecurityConstraintMapping" />
<ref bean="securityConstraintMapping" />
</list>
</property>
<property name="handler" ref="secHandlerCollection" />
</bean>
<bean id="contexts" class="org.eclipse.jetty.server.handler.ContextHandlerCollection">
</bean>
<bean id="jettyPort" class="org.apache.activemq.web.WebConsolePort" init-method="start">
<!-- the default port number for the web console -->
<property name="host" value="0.0.0.0"/>
<property name="port" value="8161"/>
</bean>
<bean id="Server" depends-on="jettyPort" class="org.eclipse.jetty.server.Server"
destroy-method="stop">
<property name="handler">
<bean id="handlers" class="org.eclipse.jetty.server.handler.HandlerCollection">
<property name="handlers">
<list>
<ref bean="contexts" />
<ref bean="securityHandler" />
</list>
</property>
</bean>
</property>
</bean>
<bean id="invokeConnectors" class="org.springframework.beans.factory.config.MethodInvokingFactoryBean">
<property name="targetObject" ref="Server" />
<property name="targetMethod" value="setConnectors" />
<property name="arguments">
<list>
<bean id="Connector" class="org.eclipse.jetty.server.ServerConnector">
<constructor-arg ref="Server" />
<!-- see the jettyPort bean -->
<property name="host" value="#{systemProperties['jetty.host']}" />
<property name="port" value="#{systemProperties['jetty.port']}" />
</bean>
<!--
Enable this connector if you wish to use https with web console
-->
<!-- Enabling the following connector on Nov 11,2020 for HTTPS -->
<bean id="SecureConnector" class="org.eclipse.jetty.server.ServerConnector">
<constructor-arg ref="Server" />
<constructor-arg>
<bean id="handlers" class="org.eclipse.jetty.util.ssl.SslContextFactory">
<property name="keyStorePath" value="${activemq.conf}/broker.ks" />
<property name="keyStorePassword" value="password" />
</bean>
</constructor-arg>
<property name="port" value="8162" />
</bean>
</list>
</property>
</bean>
<bean id="configureJetty" class="org.springframework.beans.factory.config.MethodInvokingFactoryBean">
<property name="staticMethod" value="org.apache.activemq.web.config.JspConfigurer.configureJetty" />
<property name="arguments">
<list>
<ref bean="Server" />
<ref bean="secHandlerCollection" />
</list>
</property>
</bean>
<bean id="invokeStart" class="org.springframework.beans.factory.config.MethodInvokingFactoryBean"
depends-on="configureJetty, invokeConnectors">
<property name="targetObject" ref="Server" />
<property name="targetMethod" value="start" />
</bean>
</beans>

Apache Ignite Join Query Returns Incorrect Results Due to Data not Being Co-located

I'm new to Ignite and following various blogs to understand the configuration required.
As a starter, I have used "movie lens" data (Movies & Ratings) and used Ignite Web Console to generate the required configuration.
Please find the required details about my code below:
1) MySQL Table Structure
CREATE TABLE movies( movie_id INTEGER,
movie_name VARCHAR(250), genre VARCHAR(250),
CONSTRAINT pk_movie PRIMARY KEY (movie_id) );
CREATE TABLE ratings(
user_id INTEGER, movie_id INTEGER, rating FLOAT, timestamp TIMESTAMP,
CONSTRAINT pk_rating PRIMARY KEY (user_id, movie_id) );
2) Ignite Config File in Spring Boot Project (Auto Generated by Ignite Web Console)
<?xml version="1.0" encoding="UTF-8"?>
<!-- This file was generated by Ignite Web Console (05/15/2019, 18:37) -->
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:util="http://www.springframework.org/schema/util"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/util
http://www.springframework.org/schema/util/spring-util.xsd">
<!-- Load external properties file. -->
<bean id="placeholderConfig" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
<property name="location" value="classpath:secret.properties"/>
</bean>
<!-- Data source beans will be initialized from external properties file. -->
<bean id="dsGeneric_Movielens" class="com.mchange.v2.c3p0.ComboPooledDataSource">
<property name="jdbcUrl" value="${dsGeneric_Movielens.jdbc.url}"/>
<property name="user" value="${dsGeneric_Movielens.jdbc.username}"/>
<property name="password" value="${dsGeneric_Movielens.jdbc.password}"/>
</bean>
<bean class="org.apache.ignite.configuration.IgniteConfiguration">
<property name="igniteInstanceName" value="ImportedCluster"/>
<property name="discoverySpi">
<bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
<property name="ipFinder">
<bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinder">
<property name="addresses">
<list>
<value>127.0.0.1:47500..47510</value>
</list>
</property>
</bean>
</property>
</bean>
</property>
<property name="cacheConfiguration">
<list>
<bean class="org.apache.ignite.configuration.CacheConfiguration">
<property name="name" value="MoviesCache"/>
<property name="cacheMode" value="REPLICATED"/>
<property name="atomicityMode" value="ATOMIC"/>
<property name="cacheStoreFactory">
<bean class="org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreFactory">
<property name="dataSourceBean" value="dsGeneric_Movielens"/>
<property name="dialect">
<bean class="org.apache.ignite.cache.store.jdbc.dialect.BasicJdbcDialect">
</bean>
</property>
<property name="types">
<list>
<bean class="org.apache.ignite.cache.store.jdbc.JdbcType">
<property name="cacheName" value="MoviesCache"/>
<property name="keyType" value="java.lang.Integer"/>
<property name="valueType" value="com.learnwithmanoj.ignite.model.Movies"/>
<property name="databaseSchema" value="movielens"/>
<property name="databaseTable" value="movies"/>
<property name="keyFields">
<list>
<bean class="org.apache.ignite.cache.store.jdbc.JdbcTypeField">
<constructor-arg>
<util:constant static-field="java.sql.Types.INTEGER"/>
</constructor-arg>
<constructor-arg value="movie_id"/>
<constructor-arg value="int"/>
<constructor-arg value="movieId"/>
</bean>
</list>
</property>
<property name="valueFields">
<list>
<bean class="org.apache.ignite.cache.store.jdbc.JdbcTypeField">
<constructor-arg>
<util:constant static-field="java.sql.Types.VARCHAR"/>
</constructor-arg>
<constructor-arg value="movie_name"/>
<constructor-arg value="java.lang.String"/>
<constructor-arg value="movieName"/>
</bean>
<bean class="org.apache.ignite.cache.store.jdbc.JdbcTypeField">
<constructor-arg>
<util:constant static-field="java.sql.Types.VARCHAR"/>
</constructor-arg>
<constructor-arg value="genre"/>
<constructor-arg value="java.lang.String"/>
<constructor-arg value="genre"/>
</bean>
</list>
</property>
</bean>
</list>
</property>
</bean>
</property>
<property name="readThrough" value="true"/>
<property name="writeThrough" value="true"/>
<property name="queryEntities">
<list>
<bean class="org.apache.ignite.cache.QueryEntity">
<property name="keyType" value="java.lang.Integer"/>
<property name="valueType" value="com.learnwithmanoj.ignite.model.Movies"/>
<property name="keyFieldName" value="movieId"/>
<property name="keyFields">
<list>
<value>movieId</value>
</list>
</property>
<property name="fields">
<map>
<entry key="movieName" value="java.lang.String"/>
<entry key="genre" value="java.lang.String"/>
<entry key="movieId" value="java.lang.Integer"/>
</map>
</property>
<property name="aliases">
<map>
<entry key="movieId" value="movie_id"/>
<entry key="movieName" value="movie_name"/>
</map>
</property>
</bean>
</list>
</property>
</bean>
<bean class="org.apache.ignite.configuration.CacheConfiguration">
<property name="name" value="RatingsCache"/>
<property name="cacheMode" value="REPLICATED"/>
<property name="atomicityMode" value="ATOMIC"/>
<property name="cacheStoreFactory">
<bean class="org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreFactory">
<property name="dataSourceBean" value="dsGeneric_Movielens"/>
<property name="dialect">
<bean class="org.apache.ignite.cache.store.jdbc.dialect.BasicJdbcDialect">
</bean>
</property>
<property name="types">
<list>
<bean class="org.apache.ignite.cache.store.jdbc.JdbcType">
<property name="cacheName" value="RatingsCache"/>
<property name="keyType" value="com.learnwithmanoj.ignite.model.RatingsKey"/>
<property name="valueType" value="com.learnwithmanoj.ignite.model.Ratings"/>
<property name="databaseSchema" value="movielens"/>
<property name="databaseTable" value="ratings"/>
<property name="keyFields">
<list>
<bean class="org.apache.ignite.cache.store.jdbc.JdbcTypeField">
<constructor-arg>
<util:constant static-field="java.sql.Types.INTEGER"/>
</constructor-arg>
<constructor-arg value="user_id"/>
<constructor-arg value="int"/>
<constructor-arg value="userId"/>
</bean>
<bean class="org.apache.ignite.cache.store.jdbc.JdbcTypeField">
<constructor-arg>
<util:constant static-field="java.sql.Types.INTEGER"/>
</constructor-arg>
<constructor-arg value="movie_id"/>
<constructor-arg value="int"/>
<constructor-arg value="movieId"/>
</bean>
</list>
</property>
<property name="valueFields">
<list>
<bean class="org.apache.ignite.cache.store.jdbc.JdbcTypeField">
<constructor-arg>
<util:constant static-field="java.sql.Types.REAL"/>
</constructor-arg>
<constructor-arg value="rating"/>
<constructor-arg value="java.lang.Double"/>
<constructor-arg value="rating"/>
</bean>
<bean class="org.apache.ignite.cache.store.jdbc.JdbcTypeField">
<constructor-arg>
<util:constant static-field="java.sql.Types.TIMESTAMP"/>
</constructor-arg>
<constructor-arg value="timestamp"/>
<constructor-arg value="java.sql.Timestamp"/>
<constructor-arg value="timestamp"/>
</bean>
</list>
</property>
</bean>
</list>
</property>
</bean>
</property>
<property name="readThrough" value="true"/>
<property name="writeThrough" value="true"/>
<property name="queryEntities">
<list>
<bean class="org.apache.ignite.cache.QueryEntity">
<property name="keyType" value="com.learnwithmanoj.ignite.model.RatingsKey"/>
<property name="valueType" value="com.learnwithmanoj.ignite.model.Ratings"/>
<property name="keyFields">
<list>
<value>userId</value>
<value>movieId</value>
</list>
</property>
<property name="fields">
<map>
<entry key="userId" value="java.lang.Integer"/>
<entry key="movieId" value="java.lang.Integer"/>
<entry key="rating" value="java.lang.Double"/>
<entry key="timestamp" value="java.sql.Timestamp"/>
</map>
</property>
<property name="aliases">
<map>
<entry key="userId" value="user_id"/>
<entry key="movieId" value="movie_id"/>
</map>
</property>
<property name="indexes">
<list>
<bean class="org.apache.ignite.cache.QueryIndex">
<property name="name" value="fk_movie_id"/>
<property name="indexType" value="SORTED"/>
<property name="fields">
<map>
<entry key="movieId" value="false"/>
</map>
</property>
</bean>
</list>
</property>
</bean>
</list>
</property>
</bean>
</list>
</property>
</bean>
</beans>
3) Ignite Config File in Ignite Installation Folder
<?xml version="1.0" encoding="UTF-8"?>
<!-- This file was generated by Ignite Web Console (05/15/2019, 08:52) -->
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:util="http://www.springframework.org/schema/util"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/util
http://www.springframework.org/schema/util/spring-util.xsd">
<!-- Load external properties file. -->
<bean id="placeholderConfig" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
<property name="location" value="classpath:secret.properties"/>
</bean>
<!-- Data source beans will be initialized from external properties file. -->
<bean id="datasource" class="com.mchange.v2.c3p0.ComboPooledDataSource">
<property name="jdbcUrl" value="${datasource.jdbc.url}"/>
<property name="user" value="${datasource.jdbc.username}"/>
<property name="password" value="${datasource.jdbc.password}"/>
</bean>
<bean class="org.apache.ignite.configuration.IgniteConfiguration">
<property name="igniteInstanceName" value="MovieRatingsCluster"/>
<property name="discoverySpi">
<bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
<property name="ipFinder">
<bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinder">
<property name="addresses">
<list>
<value>127.0.0.1:47500..47510</value>
</list>
</property>
</bean>
</property>
</bean>
</property>
</bean>
</beans>
Problem Statement:
When I start my Spring Boot Application as a stand-alone Ignite node, then all the data from the MySQL database is getting synced into Ignite node as expected. Also, if I try to join between the Ratings and Movies table, it's joining properly and giving the correct result.
Join Query Used:
select r.movie_id, m.movie_name, count(r.rating) as count from
"RatingsCache".ratings r inner join "MoviesCache".movies m where
r.movie_id = m.movie_id group by r.movie_id order by count desc limit
5;
Now coming to the problem. If I do the same testing using more than one node, then the Join starts to give incorrect results due to the data not being colocated.
Can anyone guide me with the right set of configuration which needs to be added to fix the Join problem in Partition Mode?
I have even tried to add the below config.
<property name="cacheKeyConfiguration">
<list>
<bean class="org.apache.ignite.cache.CacheKeyConfiguration">
<property name="typeName" value="com.learnwithmanoj.ignite.model.RatingsKey" />
<property name="affinityKeyFieldName" value="movie_id" />
</bean>
</list>
</property>
But got the below error.
Caused by: org.apache.ignite.binary.BinaryObjectException: Binary type
has different affinity key fields
[typeName=com.learnwithmanoj.ignite.model.RatingsKey,
affKeyFieldName1=null, affKeyFieldName2=movie_id]
Your solution should work. First you need to clean marshaller/ directory under Ignite home dir on all nodes, then restart those nodes.
You might also need to specify key field name in caps: MOVIE_ID.

ActiveMQ Web console using LDAP Active Directory authentication

Struggling to get ActiveMQ web console to use LDAP and authenticated against Active Directory.
No errors when starting MQ, username/password login box prompt appears but doesn't progress when inserting the correct credentials.
Version
5.15.6
login.config
amqLdapLoginModule {
org.eclipse.jetty.jaas.spi.LdapLoginModule required
debug="true"
contextFactory="com.sun.jndi.ldap.LdapCtxFactory"
hostname="ad-server1.domain.com"
port="389"
bindDn="CN=readonly-user,OU=Accounts,DC=domain,DC=com"
bindPassword="readonly-user-password"
authenticationMethod="simple"
forceBindingLogin="false"
userBaseDn="CN=users,DC=domain,DC=com"
userRdnAttribute="uid"
userIdAttribute="uid"
userPasswordAttribute="userPassword"
userObjectClass="inetOrgPerson"
roleBaseDn="CN=groups,DC=domain,dc=com"
roleNameAttribute="cn"
roleMemberAttribute="uniqueMember"
roleObjectClass="groupOfUniqueNames";
};
jetty.xml
<bean id="ldapLoginService" class="org.eclipse.jetty.jaas.JAASLoginService">
<property name="name" value="LdapRealm" />
<property name="loginModuleName" value="amqLdapLoginModule" />
<property name="roleClassNames" value="org.eclipse.jetty.jaas.JAASRole" />
<property name="identityService" ref="identityService" />
</bean>
<bean id="identityService" class="org.eclipse.jetty.security.DefaultIdentityService"/>
<bean id="securityConstraint" class="org.eclipse.jetty.util.security.Constraint">
<property name="name" value="BASIC" />
<property name="roles" value="admins-group" />
<!-- set authenticate=false to disable login -->
<property name="authenticate" value="true" />
</bean>
<bean id="adminSecurityConstraint" class="org.eclipse.jetty.util.security.Constraint">
<property name="name" value="BASIC" />
<property name="roles" value="admins-group" />
<!-- set authenticate=false to disable login -->
<property name="authenticate" value="true" />
</bean>
<bean id="securityHandlerLdap" class="org.eclipse.jetty.security.ConstraintSecurityHandler">
<property name="loginService" ref="ldapLoginService" />
<property name="identityService" ref="identityService" />
<property name="realmName" value="LdapRealm" />
<property name="authenticator">
<bean class="org.eclipse.jetty.security.authentication.BasicAuthenticator" />
</property>
<property name="constraintMappings">
<list>
<ref bean="adminSecurityConstraintMapping" />
<ref bean="securityConstraintMapping" />
</list>
</property>
<property name="handler" ref="secHandlerCollection" />
</bean>
<bean id="contexts" class="org.eclipse.jetty.server.handler.ContextHandlerCollection">
</bean>
Following the guide from http://bacedifo.blogspot.com/2013/06/securing-activemq-580-web-console-using.html and using the ldaptive ldap java library with some tweaks to the configs I managed to get this working for our AD environment.
Copied the ldaptive-{version number}.jar and jetty-jass-{version number}.jar to /activemq/lib directory.
login.conf
activemq {
org.ldaptive.jaas.LdapLoginModule required
debug=true
storePass="true"
ldapUrl="ldap://ldap-server1.domainname.com:389 ldap://ldap-server2.domainname.com:389"
connectionStrategy="ACTIVE_PASSIVE"
bindDn="CN=ldap-readaccount,OU=Read Accounts,DC=domainname,DC=com"
baseDn="OU=accounts,DC=domainname,DC=com"
bindCredential="ldapuser-password"
useStartTLS="false"
userFilter="(sAMAccountName={user})";
org.ldaptive.jaas.LdapRoleAuthorizationModule required
useFirstPass="true"
ldapUrl="ldap://ldap-server1.domainname.com:389 ldap://ldap-server2.domainname.com:389"
connectionStrategy="ACTIVE_PASSIVE"
bindDn="CN=ldap-readaccount,OU=Read Accounts,DC=domainname,DC=com"
baseDn="OU=groups,DC=domainname,DC=com"
bindCredential="ldapuser-password"
roleFilter="(&(cn=webconsoleadmins)(member={user}))"
useStartTLS="false"
defaultRole="admins"
roleAttribute="cn";
};
jetty.xml
<beans xmlns="http://www.springframework.org/schema/beans" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
<bean id="securityLoginService" class="org.eclipse.jetty.jaas.JAASLoginService">
<property name="name" value="LdapRealm" />
<property name="loginModuleName" value="activemq" />
<property name="roleClassNames" value="org.ldaptive.jaas.LdapRole" />
<property name="identityService" ref="identityService" />
</bean>
<bean id="identityService" class="org.eclipse.jetty.security.DefaultIdentityService"/>
<bean id="securityConstraint" class="org.eclipse.jetty.util.security.Constraint">
<property name="name" value="BASIC" />
<property name="roles" value="admins,webconsoleadmins" />
<!-- set authenticate=false to disable login -->
<property name="authenticate" value="true" />
</bean>
<bean id="adminSecurityConstraint" class="org.eclipse.jetty.util.security.Constraint">
<property name="name" value="BASIC" />
<property name="roles" value="admins,webconsoleadmins" />
<!-- set authenticate=false to disable login -->
<property name="authenticate" value="true" />
</bean>
...
<bean id="securityHandler" class="org.eclipse.jetty.security.ConstraintSecurityHandler">
<property name="loginService" ref="securityLoginService" />
<property name="identityService" ref="identityService" />
<property name="authenticator">
<bean class="org.eclipse.jetty.security.authentication.BasicAuthenticator" />
</property>
<property name="constraintMappings">
<list>
<ref bean="adminSecurityConstraintMapping" />
<ref bean="securityConstraintMapping" />
</list>
</property>
<property name="handler" ref="secHandlerCollection" />
</bean>

Ignite DataStreamer not loading data

I'm using ignite version 2.0.0
I am trying to load about 40M data from db to my ignite instances(2 in this case). I read about the Ignite DataStreamers from https://apacheignite.readme.io/v1.2/docs/data-streamers
Because of the very low data loading speed of loadCache() i changed loadCache() with IgniteDataStreamer.addData().
Upon execution, i notice in the web console that the metrics for on-heap entries gets incrementing(i.e., it shows that data is being loaded). But when i query the ignite cache, i get the result as empty.
Also i notice that the server logs are showcasing this exception:
[01:46:19,541][ERROR][flusher-0-#66%RemoteIgniteCluster%][GridCacheWriteBehindStore] Unable to update underlying store: CacheJdbcPojoStore []
javax.cache.CacheException: Failed to read property value from non binary object [class=class java.lang.Integer, property=class_no]
at org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStore.extractBinaryParameter(CacheJdbcPojoStore.java:122)
at org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStore.extractParameter(CacheJdbcPojoStore.java:69)
at org.apache.ignite.cache.store.jdbc.CacheAbstractJdbcStore.fillValueParameters(CacheAbstractJdbcStore.java:1414)
at org.apache.ignite.cache.store.jdbc.CacheAbstractJdbcStore.writeAll(CacheAbstractJdbcStore.java:1081)
at org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStore.updateStore(GridCacheWriteBehindStore.java:804)
at org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStore.applyBatch(GridCacheWriteBehindStore.java:720)
at org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStore.access$2400(GridCacheWriteBehindStore.java:75)
at org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStore$Flusher.flushCacheCoalescing(GridCacheWriteBehindStore.java:1108)
at org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStore$Flusher.body(GridCacheWriteBehindStore.java:1006)
at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110)
at java.lang.Thread.run(Thread.java:745)
[01:50:00,945][ERROR][flusher-0-#66%RemoteIgniteCluster%][GridCacheWriteBehindStore] Unable to update underlying store: CacheJdbcPojoStore []
javax.cache.integration.CacheWriterException: Failed to write entries in database
at org.apache.ignite.cache.store.jdbc.CacheAbstractJdbcStore.writeAll(CacheAbstractJdbcStore.java:1151)
at org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStore.updateStore(GridCacheWriteBehindStore.java:804)
at org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStore.applyBatch(GridCacheWriteBehindStore.java:720)
at org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStore.access$2400(GridCacheWriteBehindStore.java:75)
at org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStore$Flusher.flushCacheCoalescing(GridCacheWriteBehindStore.java:1108)
at org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStore$Flusher.body(GridCacheWriteBehindStore.java:1006)
at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110)
at java.lang.Thread.run(Thread.java:745)
My LoadCache.java code is:
package load;
import javax.naming.event.ObjectChangeListener;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteDataStreamer;
import org.apache.ignite.Ignition;
public class LoadCaches {
public static void main(String[] args) throws Exception {
try (Ignite ignite = Ignition.start("RemoteIgniteCluster-client.xml");IgniteDataStreamer<Integer, Object> stmr = ignite.dataStreamer("PersonsCache")) {
System.out.println(">>> Loading caches...");
stmr.allowOverwrite(true);
stmr.autoFlushFrequency(1000);
System.out.println(">>> Loading cache: PersonsCache");
for (int i = 0; i < 5000000; i++)
stmr.addData(i, new Integer(i));
/*ignite.cache("PersonsCache").loadCache(null);*/
System.out.println(">>> All caches loaded!");
}
}
}
I tried changing
IgniteDataStreamer<Integer, Object> stmr = ignite.dataStreamer("PersonsCache"))
and
stmr.addData(i, new Integer(i));
with
IgniteDataStreamer<Integer, String> stmr = ignite.dataStreamer("PersonsCache"))
and
stmr.addData(i, Integer.toString(i));
and yet i get the same exception in the server logs, and when i try to query the cache i get empty results.
My spring xmls look like this:
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:util="http://www.springframework.org/schema/util"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/util
http://www.springframework.org/schema/util/spring-util.xsd">
<!-- Load external properties file. -->
<bean id="placeholderConfig" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
<property name="location" value="classpath:secret.properties"/>
</bean>
<!-- Data source beans will be initialized from external properties file. -->
<bean id="dsMySQL_DB" class="com.mysql.jdbc.jdbc2.optional.MysqlDataSource">
<property name="URL" value="${dsMySQL_DB.jdbc.url}"/>
<property name="user" value="${dsMySQL_DB.jdbc.username}"/>
<property name="password" value="${dsMySQL_DB.jdbc.password}"/>
</bean>
<bean class="org.apache.ignite.configuration.IgniteConfiguration">
<property name="igniteInstanceName" value="testcluster"/>
<property name="peerClassLoadingEnabled" value="false" />
<property name="includeEventTypes">
<list>
<util:constant static-field="org.apache.ignite.events.EventType.EVT_TASK_STARTED" />
<util:constant
static-field="org.apache.ignite.events.EventType.EVT_TASK_FINISHED" />
<util:constant static-field="org.apache.ignite.events.EventType.EVT_TASK_FAILED" />
</list>
</property>
<!-- Configure internal thread pool. -->
<property name="publicThreadPoolSize" value="64" />
<!-- Configure system thread pool. -->
<property name="systemThreadPoolSize" value="32" />
<property name="discoverySpi">
<bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
<property name="ipFinder">
<bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinder">
<!-- <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder"> -->
<property name="addresses">
<list>
<value>127.0.0.1:47500..47510</value>
</list>
</property>
</bean>
</property>
</bean>
</property>
<property name="atomicConfiguration">
<bean class="org.apache.ignite.configuration.AtomicConfiguration">
<property name="backups" value="0"/>
</bean>
</property>
<property name="cacheKeyConfiguration">
<list>
<bean class="org.apache.ignite.cache.CacheKeyConfiguration">
<constructor-arg value="com.gmail.testcluster.model.Persons"/>
<constructor-arg value="age"/>
</bean>
</list>
</property>
<property name="cacheConfiguration">
<list>
<bean class="org.apache.ignite.configuration.CacheConfiguration">
<!-- Set rebalance batch size to 1 MB. -->
<property name="rebalanceBatchSize" value="#{1024 * 1024}" />
<!-- Explicitly disable rebalance throttling. -->
<property name="rebalanceThrottle" value="0" />
<!-- Set 4 threads for rebalancing. -->
<property name="rebalanceThreadPoolSize" value="4" />
<property name="name" value="PersonsCache"/>
<property name="cacheMode" value="PARTITIONED"/>
<property name="atomicityMode" value="ATOMIC"/>
<property name="backups" value="0"/>
<property name="affinity">
<bean class="org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction">
<property name="partitions" value="2048"/>
</bean>
</property>
<!-- <property name="queryDetailMetricsSize" value="50"/> -->
<property name="cacheStoreFactory">
<bean class="org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreFactory">
<property name="dataSourceBean" value="dsMySQL_DB"/>
<property name="dialect">
<bean class="org.apache.ignite.cache.store.jdbc.dialect.MySQLDialect">
</bean>
</property>
<property name="types">
<list>
<bean class="org.apache.ignite.cache.store.jdbc.JdbcType">
<property name="cacheName" value="PersonsCache"/>
<property name="keyType" value="java.lang.Integer"/>
<property name="valueType" value="com.gmail.testcluster.model.Persons"/>
<property name="databaseSchema" value="MY_DB"/>
<property name="databaseTable" value="PERSONS"/>
<property name="keyFields">
<list>
<bean class="org.apache.ignite.cache.store.jdbc.JdbcTypeField">
<constructor-arg>
<util:constant static-field="java.sql.Types.INTEGER"/>
</constructor-arg>
<constructor-arg value="ID"/>
<constructor-arg value="int"/>
<constructor-arg value="id"/>
</bean>
</list>
</property>
<property name="valueFields">
<list>
<bean class="org.apache.ignite.cache.store.jdbc.JdbcTypeField">
<constructor-arg>
<util:constant static-field="java.sql.Types.INTEGER"/>
</constructor-arg>
<constructor-arg value="CLASS_NO"/>
<constructor-arg value="java.lang.Integer"/>
<constructor-arg value="class_no"/>
</bean>
<bean class="org.apache.ignite.cache.store.jdbc.JdbcTypeField">
<constructor-arg>
<util:constant static-field="java.sql.Types.VARCHAR"/>
</constructor-arg>
<constructor-arg value="NAME"/>
<constructor-arg value="java.lang.String"/>
<constructor-arg value="name"/>
</bean>
<bean class="org.apache.ignite.cache.store.jdbc.JdbcTypeField">
<constructor-arg>
<util:constant static-field="java.sql.Types.INTEGER"/>
</constructor-arg>
<constructor-arg value="AGE"/>
<constructor-arg value="java.lang.Integer"/>
<constructor-arg value="age"/>
</bean>
<bean class="org.apache.ignite.cache.store.jdbc.JdbcTypeField">
<constructor-arg>
<util:constant static-field="java.sql.Types.DOUBLE"/>
</constructor-arg>
<constructor-arg value="R_AMT"/>
<constructor-arg value="java.lang.Double"/>
<constructor-arg value="rAmt"/>
</bean>
</list>
</property>
</bean>
</list>
</property>
</bean>
</property>
<property name="readThrough" value="true"/>
<property name="writeThrough" value="true"/>
<property name="writeBehindEnabled" value="true"/>
<property name="writeBehindBatchSize" value="500"/>
<property name="writeBehindFlushSize" value="1000"/>
<property name="writeBehindFlushFrequency" value="60000"/>
<property name="queryEntities">
<list>
<bean class="org.apache.ignite.cache.QueryEntity">
<property name="keyType" value="java.lang.Integer"/>
<property name="valueType" value="com.gmail.testcluster.model.Persons"/>
<property name="fields">
<map>
<entry key="class_no" value="java.lang.Integer"/>
<entry key="name" value="java.lang.String"/>
<entry key="age" value="java.lang.Integer"/>
<entry key="rAmt" value="java.lang.Double"/>
</map>
</property>
<property name="aliases">
<map>
<entry key="rAmt" value="R_AMT"/>
</map>
</property>
<!-- <property name="indexes">
<list>
<bean class="org.apache.ignite.cache.QueryIndex">
<property name="name" value="queryindexing"/>
<property name="indexType" value="SORTED"/>
<property name="fields">
<map>
<entry key="class_no" value="true"/>
</map>
</property>
</bean>
</list>
</property> -->
</bean>
</list>
</property>
</bean>
</list>
</property>
</bean>
I am not sure where the problem is, can someone take a look? Thanks!!!
When DataStreamer's property allowOverwrite=false (default) it causes Persistent Store to be skipped.
Cache store is invoked only when AllowOverwrite is true. Link to doc
if you want to use readThrough, then use writeThrough. You can read about these modes in doc

My jasig don't use the User ldap password

We have several services auntentican with an LDAP server, we put JASIG as the main entrance, we deploy JASIG on Jboss7 and works well over SSL, however our LDAP server over TLS works. My problem is that over TLS for any authenticated user runs any user password, right or wrong.
I do not know why the login username is sensitive and not sensitive to the password, pass any password.
Thank you.
Show parts interesting parts of my DeployConfigContex.xml file.
This is our AuthenticationManager:
<bean id="authenticationManager"
class="org.jasig.cas.authentication.AuthenticationManagerImpl">
<property name="credentialsToPrincipalResolvers">
<list>
<bean
class="org.jasig.cas.authentication.principal.CredentialsToLDAPAttributePrincipalResolver">
<property name="credentialsToPrincipalResolver">
<bean
class="org.jasig.cas.authentication.principal.UsernamePasswordCredentialsToPrincipalResolver" />
</property>
<property name="filter" value="(cn=%u)" />
<property name="principalAttributeName" value="cn" />
<property name="searchBase" value="ou=ServiciosCentrales,ou=Usuarios,dc=educacion,dc=es" />
<property name="contextSource" ref="contextSource" />
<property name="attributeRepository">
<ref bean="attributeRepository" />
</property>
</bean>
</list>
</property>
<property name="authenticationHandlers">
<list>
<bean class="org.jasig.cas.authentication.handler.support.HttpBasedServiceCredentialsAuthenticationHandler"
p:httpClient-ref="httpClient" />
<bean class="org.jasig.cas.adaptors.ldap.BindLdapAuthenticationHandler">
<property name="filter" value="cn=%u" />
<property name="searchBase" value="ou=ServiciosCentrales,ou=Usuarios,dc=educacion,dc=es" />
<property name="contextSource" ref="contextSource" />
<property name="ignorePartialResultException" value="yes" />
</bean>
</list>
</property>
</bean>
this es our contexsource:
<property name="pooled" value="false" />
<property name="userDn" value="cn=secret,dc=educacion,dc=es"/>
<property name="password" value="secret"/>
<property name="url" value="ldap://192.168.0.100:389"/>
<qualifier value="secured"/>
<property name="authenticationStrategy">
<bean class="org.springframework.ldap.core.support.DefaultTlsDirContextAuthenticationStrategy">
<property name="sslSocketFactory">
<bean class="com.cein.educa.ldap.LDAPSocketFactory" />
</property>
<property name="hostnameVerifier">
<bean class="com.cein.educa.ldap.LDAPHostNameVerifier" />
</property>
<property name="shutdownTlsGracefully" value="true" />
</bean>
</property>
<property name="baseEnvironmentProperties">
<map>
<entry key="java.naming.security.authentication" value="simple" />
</map>
</property>
</bean>