How to properly set up XGBoost4J training? - xgboost

I am trying to follow the example code from XGBoost4J's tutorial, but am running into a check failure in the C++ implementation. The error message is not obvious what's wrong:
ml.dmlc.xgboost4j.java.XGBoostError: [21:28:54] /Users/phcho/tmp/xgboost/src/objective/regression_obj.cu:35: Check failed: info.labels.Shape(0) == info.num_row_ (0 vs. 329) : Invalid shape of labels.
Stack trace:
[bt] (0) 1 libxgboost4j3410071582095733624.dyl 0x000000013ac09a74 dmlc::LogMessageFatal::~LogMessageFatal() + 116
[bt] (1) 2 libxgboost4j3410071582095733624.dyl 0x000000013ad495b9 xgboost::obj::(anonymous namespace)::CheckRegInputs(xgboost::MetaInfo const&, xgboost::HostDeviceVector<float> const&) + 249
[bt] (2) 3 libxgboost4j3410071582095733624.dyl 0x000000013ad4dda8 xgboost::obj::RegLossObj<xgboost::obj::LogisticClassification>::GetGradient(xgboost::HostDeviceVector<float> const&, xgboost::MetaInfo const&, int, xgboost::HostDeviceVector<xgboost::detail::GradientPairInternal<float> >*) + 40
[bt] (3) 4 libxgboost4j3410071582095733624.dyl 0x000000013ace6f04 xgboost::LearnerImpl::UpdateOneIter(int, std::__1::shared_ptr<xgboost::DMatrix>) + 580
[bt] (4) 5 libxgboost4j3410071582095733624.dyl 0x000000013ac14b89 XGBoosterUpdateOneIter + 137
[bt] (5) 6 ??? 0x000000011671978c 0x0 + 4671510412
[bt] (6) 7 ??? 0x0000000116712b28 0x0 + 4671482664
[bt] (7) 8 ??? 0x0000000116712e90 0x0 + 4671483536
[bt] (8) 9 ??? 0x00000001167129ba 0x0 + 4671482298
The code:
List<float[]> rows = new ArrayList<>();
int trainRows = (int) (rows.size() * 0.7);
int testRows = rows.size() - trainRows;
int columnCount = rows.get(0).length;
float[] flatTrainMatrix = new float[trainRows * columnCount];
float[] flatTestMatrix = new float[testRows * columnCount];
for (int i = 0; i < rows.size(); i++) {
if (i < trainRows) {
System.arraycopy(rows.get(i), 0, flatTrainMatrix, i * columnCount, columnCount);
} else {
System.arraycopy(rows.get(i), 0, flatTestMatrix, (i - trainRows) * columnCount, columnCount);
}
}
var params = Map.<String, Object>of("max_depth", 4, "eta", 1.0, "objective", "binary:logistic", "eval_metric", "auc");
try {
DMatrix trainMatrix = new DMatrix(flatTrainMatrix, trainRows, columnCount, -1);
DMatrix testMatrix = new DMatrix(flatTestMatrix, testRows, columnCount, -1);
var booster = XGBoost.train(trainMatrix, params, 1, Map.of("train", trainMatrix, "test", testMatrix), null, null);
booster.saveModel("/tmp/model.bin");
} catch (XGBoostError e) {
throw new RuntimeException(e); // The error message from above is from here.
}
}
I am using the ml.dmlc:xgboost4j_2.12:1.6.1.

Related

Vulkan: Loading floating point cubemap textures distorted

I am using vulkan-tutorial codes and i made modify for cubemap.
when i use VK_FORMAT_R8G8B8A8_UNORM is working with this code:
unsigned char* pixelsArray[6];
for (int i = 0; i < 6; ++i)
{
pixelsArray[i] = stbi_load(imageFileArray[i].c_str(), &texWidth, &texHeight, &texChannels, STBI_rgb_alpha);
}
VkDeviceSize allSize = texWidth * texHeight * 4 * 6;
VkDeviceSize size = texWidth * texHeight * 4 ;
VkBufferCreateInfo bufferInfo{};
...
bufferInfo.size = allSize ;
vkMapMemory(device, stagingBufferMemory, 0, AllSize, 0, &data);
for(int i = 0; i < 6; ++i)
{
memcpy( (char*) data + (size*i) , pixelsArray[i], static_cast<size_t>(size));
}
vkUnmapMemory(device, stagingBufferMemory);
VkImageCreateInfo imageInfo{};
...
imageInfo.arrayLayers = 6;
imageInfo.format = VK_FORMAT_R8G8B8A8_UNORM;
imageInfo.flags = VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT;
VkImageViewCreateInfo viewInfo{};
...
viewInfo.viewType = VK_IMAGE_VIEW_TYPE_CUBE;
viewInfo.format = VK_FORMAT_R8G8B8A8_UNORM;
viewInfo.subresourceRange.layerCount = 6;
but when i try VK_FORMAT_R16G16B16A16_SFLOAT is giving distorted display and no validation error with this code:
float* pixelsArray[6];
for (int i = 0; i < 6; ++i)
{
pixelsArray[i] = stbi_loadf(imageFileArray[i].c_str(), &texWidth, &texHeight, &texChannels, STBI_rgb_alpha);
}
VkDeviceSize allSize = texWidth * texHeight * 4 * 6 * 2;// I added *2
VkDeviceSize size = texWidth * texHeight * 4 * 2;// I added *2
VkBufferCreateInfo bufferInfo{};
...
bufferInfo.size = allSize ;
vkMapMemory(device, stagingBufferMemory, 0, AllSize, 0, &data);
for(int i = 0; i < 6; ++i)
{
memcpy( (char*) data + (size*i) , pixelsArray[i], static_cast<size_t>(size));
}
vkUnmapMemory(device, stagingBufferMemory);
VkImageCreateInfo imageInfo{};
...
imageInfo.arrayLayers = 6;
imageInfo.format = VK_FORMAT_R16G16B16A16_SFLOAT;
imageInfo.flags = VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT;
VkImageViewCreateInfo viewInfo{};
...
viewInfo.viewType = VK_IMAGE_VIEW_TYPE_CUBE;
viewInfo.format = VK_FORMAT_R16G16B16A16_SFLOAT;
viewInfo.subresourceRange.layerCount = 6;
when VK_FORMAT_R8G8B8A8_UNORM :
when VK_FORMAT_R16G16B16A16_SFLOAT :
i fixed the problem. problem was that i want to use half float but i was sending float to memcpy function.i searched how can i use half float and i found a solution without using extra library.
what i did add helper functions :
typedef unsigned int uint;
typedef unsigned short ushort;
uint as_uint(const float x)
{
return *(uint*)&x;
}
ushort float_to_half(const float x)
{
// IEEE-754 16-bit floating-point format (without infinity): 1-5-10, exp-15, +-131008.0, +-6.1035156E-5, +-5.9604645E-8, 3.311 digits
const uint b = as_uint(x)+0x00001000; // round-to-nearest-even: add last bit after truncated mantissa
const uint e = (b&0x7F800000)>>23; // exponent
const uint m = b&0x007FFFFF; // mantissa; in line below: 0x007FF000 = 0x00800000-0x00001000 = decimal indicator flag - initial rounding
return (b&0x80000000)>>16 | (e>112)*((((e-112)<<10)&0x7C00)|m>>13) | ((e<113)&(e>101))*((((0x007FF000+m)>>(125-e))+1)>>1) | (e>143)*0x7FFF; // sign : normalized : denormalized : saturate
}
and fix problem with this helper functions :
VkDeviceSize size_2 = texWidth * texHeight * 4;// different from the above variables in question : allSize or size
//create half float for cubemap
void* half_pixelsArray[6];
half_pixelsArray[0] = new ushort[size_2];
half_pixelsArray[1] = new ushort[size_2];
half_pixelsArray[2] = new ushort[size_2];
half_pixelsArray[3] = new ushort[size_2];
half_pixelsArray[4] = new ushort[size_2];
half_pixelsArray[5] = new ushort[size_2];
//copy from float to half float
for (int i = 0; i < 6; ++i)
{
for (int j = 0; j < size_2; ++j)
{
((ushort*)half_pixelsArray[i])[j] = float_to_half( pixelsArray[i][j] );
}
}
// and change float to half flaot in memcpy
memcpy( (char*) data + (layerSize*i) , half_pixelsArray[i], static_cast<size_t>(layerSize));

ESP-32 CAM returning pictures with a grey band on them when higher resolution than VGA

I am using ESP-32CAM (with an OV2640 camera) to send pictures to Google Drive, via HTTPS POST request to Google App Script. The payload photo is base64 format. The photo is uploaded and fully visible.
But at a higher resolution ( than VGA),the pictures are incomplete, they have a large part of them as a single color band. In Google Drive that band is grey, but when I download the photo, the color of the band is changed.
I checked if the buffer is too small using the psramFound() function, but that returned 1.
Photo upload on Google Drive : https://i.stack.imgur.com/bJwBq.jpg
Any idea how I can upload the photos correctly at the maximum resolution allowed by the board (UXGA-1600 x 1200)?
Thanks for help! :D
Code that I am using under Arduino IDE (1.8.13):
#include <WiFi.h>
#include <WiFiClientSecure.h>
#include "soc/soc.h"
#include "soc/rtc_cntl_reg.h"
#include "Base64.h"
#include "esp_camera.h"
const char* ssid = "***"; //your network SSID
const char* password = "***"; //your network password
const char* myDomain = "script.google.com";
String myScript = "/macros/s/********/exec"; //Replace with your own url
String myFilename = "filename=ESP32-CAM.jpg";
String mimeType = "&mimetype=image/jpeg";
String myImage = "&data=";
int waitingTime = 30000; //Wait 30 seconds to google response.
#define PWDN_GPIO_NUM 32
#define RESET_GPIO_NUM -1
#define XCLK_GPIO_NUM 0
#define SIOD_GPIO_NUM 26
#define SIOC_GPIO_NUM 27
#define Y9_GPIO_NUM 35
#define Y8_GPIO_NUM 34
#define Y7_GPIO_NUM 39
#define Y6_GPIO_NUM 36
#define Y5_GPIO_NUM 21
#define Y4_GPIO_NUM 19
#define Y3_GPIO_NUM 18
#define Y2_GPIO_NUM 5
#define VSYNC_GPIO_NUM 25
#define HREF_GPIO_NUM 23
#define PCLK_GPIO_NUM 22
void setup()
{
WRITE_PERI_REG(RTC_CNTL_BROWN_OUT_REG, 0);
Serial.begin(115200);
delay(10);
WiFi.mode(WIFI_STA);
Serial.println("");
Serial.print("Connecting to ");
Serial.println(ssid);
WiFi.begin(ssid, password);
while (WiFi.status() != WL_CONNECTED) {
Serial.print(".");
delay(500);
}
Serial.println("");
Serial.println("STAIP address: ");
Serial.println(WiFi.localIP());
Serial.println("");
camera_config_t config;
config.ledc_channel = LEDC_CHANNEL_0;
config.ledc_timer = LEDC_TIMER_0;
config.pin_d0 = Y2_GPIO_NUM;
config.pin_d1 = Y3_GPIO_NUM;
config.pin_d2 = Y4_GPIO_NUM;
config.pin_d3 = Y5_GPIO_NUM;
config.pin_d4 = Y6_GPIO_NUM;
config.pin_d5 = Y7_GPIO_NUM;
config.pin_d6 = Y8_GPIO_NUM;
config.pin_d7 = Y9_GPIO_NUM;
config.pin_xclk = XCLK_GPIO_NUM;
config.pin_pclk = PCLK_GPIO_NUM;
config.pin_vsync = VSYNC_GPIO_NUM;
config.pin_href = HREF_GPIO_NUM;
config.pin_sscb_sda = SIOD_GPIO_NUM;
config.pin_sscb_scl = SIOC_GPIO_NUM;
config.pin_pwdn = PWDN_GPIO_NUM;
config.pin_reset = RESET_GPIO_NUM;
config.xclk_freq_hz = 20000000;
config.pixel_format = PIXFORMAT_JPEG;
if(psramFound()){
config.frame_size = FRAMESIZE_UXGA; // FRAMESIZE_ + QVGA|CIF|VGA|SVGA|XGA|SXGA|UXGA
config.jpeg_quality = 10; //10-63 lower number means higher quality
config.fb_count = 2;
} else {
config.frame_size = FRAMESIZE_SVGA;
config.jpeg_quality = 12;
config.fb_count = 1;
}
esp_err_t err = esp_camera_init(&config);
if (err != ESP_OK) {
Serial.printf("Camera init failed with error 0x%x", err);
delay(1000);
ESP.restart();
}
sensor_t * s = esp_camera_sensor_get();
s->set_brightness(s, 0); // -2 to 2
s->set_contrast(s, 0); // -2 to 2
s->set_saturation(s, 0); // -2 to 2
s->set_special_effect(s, 0); // 0 to 6 (0 - No Effect, 1 - Negative, 2 - Grayscale, 3 - Red Tint, 4 - Green Tint, 5 - Blue Tint, 6 - Sepia)
s->set_whitebal(s, 1); // 0 = disable , 1 = enable
s->set_awb_gain(s, 1); // 0 = disable , 1 = enable
s->set_wb_mode(s, 0); // 0 to 4 - if awb_gain enabled (0 - Auto, 1 - Sunny, 2 - Cloudy, 3 - Office, 4 - Home)
s->set_exposure_ctrl(s, 1); // 0 = disable , 1 = enable
s->set_aec2(s, 0); // 0 = disable , 1 = enable
s->set_ae_level(s, 0); // -2 to 2
s->set_aec_value(s, 300); // 0 to 1200
s->set_gain_ctrl(s, 1); // 0 = disable , 1 = enable
s->set_agc_gain(s, 0); // 0 to 30
s->set_gainceiling(s, (gainceiling_t)0); // 0 to 6
s->set_bpc(s, 0); // 0 = disable , 1 = enable
s->set_wpc(s, 1); // 0 = disable , 1 = enable
s->set_raw_gma(s, 1); // 0 = disable , 1 = enable
s->set_lenc(s, 1); // 0 = disable , 1 = enable
s->set_hmirror(s, 0); // 0 = disable , 1 = enable
s->set_vflip(s, 0); // 0 = disable , 1 = enable
s->set_dcw(s, 1); // 0 = disable , 1 = enable
s->set_colorbar(s, 0); // 0 = disable , 1 = enable
s->set_framesize(s, FRAMESIZE_UXGA);
}
boolean enviar = true;
void loop() {
//if(enviar) {
saveCapturedImage();
enviar = false;
delay(60000);
//}
}
void saveCapturedImage() {
Serial.println("Connect to " + String(myDomain));
WiFiClientSecure client;
if (client.connect(myDomain, 443)) {
Serial.println("Connection successful");
camera_fb_t * fb = NULL;
fb = esp_camera_fb_get();
if(!fb) {
Serial.println("Camera capture failed");
delay(1000);
ESP.restart();
return;
}
char *input = (char *)fb->buf;
char output[base64_enc_len(3)];
String imageFile = "";
for (int i=0;i<fb->len;i++) {
base64_encode(output, (input++), 3);
if (i%3==0) imageFile += urlencode(String(output));
}
String Data = myFilename+mimeType+myImage;
esp_camera_fb_return(fb);
Serial.println("Send a captured image to Google Drive.");
client.println("POST " + myScript + " HTTP/1.1");
client.println("Host: " + String(myDomain));
client.println("Content-Length: " + String(Data.length()+imageFile.length()));
client.println("Content-Type: application/x-www-form-urlencoded");
client.println();
client.print(Data);
int Index;
for (Index = 0; Index < imageFile.length(); Index = Index+1000) {
client.print(imageFile.substring(Index, Index+1000));
}
Serial.println("Waiting for response.");
long int StartTime=millis();
while (!client.available()) {
Serial.print(".");
delay(100);
if ((StartTime+waitingTime) < millis()) {
Serial.println();
Serial.println("No response.");
//If you have no response, maybe need a greater value of waitingTime
break;
}
}
Serial.println();
while (client.available()) {
Serial.print(char(client.read()));
}
} else {
Serial.println("Connected to " + String(myDomain) + " failed.");
}
client.stop();
}
//https://github.com/zenmanenergy/ESP8266-Arduino-Examples/
String urlencode(String str)
{
String encodedString="";
char c;
char code0;
char code1;
char code2;
for (int i =0; i < str.length(); i++){
c=str.charAt(i);
if (c == ' '){
encodedString+= '+';
} else if (isalnum(c)){
encodedString+=c;
} else{
code1=(c & 0xf)+'0';
if ((c & 0xf) >9){
code1=(c & 0xf) - 10 + 'A';
}
c=(c>>4)&0xf;
code0=c+'0';
if (c > 9){
code0=c - 10 + 'A';
}
code2='\0';
encodedString+='%';
encodedString+=code0;
encodedString+=code1;
//encodedString+=code2;
}
yield();
}
return encodedString;
}

macOS weird crash in [NSString stringWithUTF8String:]

We started seeing crashes that happen when our app starts. I wasn't able to reproduce it, and it happens only to a few users.
The exception is:
Exception Type: EXC_BAD_ACCESS (SIGBUS) Exception Codes: 0x00000000
at 0x0000000105e1f32c Crashed Thread: 8
stack here
Thread 8 Crashed:
0 libsystem_platform.dylib 0x00007fff203b56f8 _os_semaphore_wait.cold.1 + 129
1 libsystem_malloc.dylib 0x00007fff20193793 szone_malloc_should_clear + 66
2 libsystem_malloc.dylib 0x00007fff201aceb7 _malloc_zone_calloc + 59
3 CoreFoundation 0x00007fff203e7b0a _CFRuntimeCreateInstance + 290
4 CoreFoundation 0x00007fff203e7289 __CFStringCreateImmutableFunnel3 + 2099
5 CoreFoundation 0x00007fff203f34fe CFStringCreateWithBytes + 27
6 Foundation 0x00007fff210eadbe +[NSString stringWithUTF8String:] + 68
7 MyApp 0x0000000104551576 +[Utility getCommandLine:] + 934
source code here
+ (void)getCommandLine:(LCProcessInfo*)process
{
int mib[3], argmax, nargs, c = 0;
char *procargs, *cp, *sp, *np;
size_t size;
mib[0] = CTL_KERN;
mib[1] = KERN_ARGMAX;
size = sizeof(argmax);
if (sysctl(mib, 2, &argmax, &size, NULL, 0) == -1)
{
ERROR(#"sysctl() of KERN_ARGMAX has failed.");
return;
}
procargs = malloc(argmax);
if (procargs == NULL)
{
ERROR(#"malloc() has failed");
return;
}
mib[0] = CTL_KERN;
mib[1] = KERN_PROCARGS2;
mib[2] = (int)process.pid;
size = argmax;
if (sysctl(mib, 3, procargs, &size, NULL, 0) == -1)
{
// Failure here means it's a system process.
process.commandLine = [NSString stringWithFormat:#"(%#)", process.name];
goto exit;
}
memcpy(&nargs, procargs, sizeof(nargs));
cp = procargs + sizeof(nargs);
for (; cp < &procargs[size]; cp++)
{
if (*cp == '\0')
{
break;
}
}
if (cp == &procargs[size])
{
goto exit;;
}
// Skip trailing '\0' characters.
for (; cp < &procargs[size]; cp++)
{
if (*cp != '\0')
{
break;
}
}
if (cp == &procargs[size])
{
goto exit;
}
// Save where argv[0] string starts.
sp = cp;
/*
* Iterate through the '\0'-terminated strings and convert '\0' to ' '
* until a string is found that has a '=' character in it (or there are
* no more strings in procargs). There is no way to deterministically
* know where the command arguments end and the environment strings
* start, which is why the '=' character is searched for as a heuristic.
*/
for (np = NULL; c < nargs && cp < &procargs[size]; cp++) {
if (*cp == '\0') {
c++;
if (np != NULL) {
/* Convert previous '\0'. */
*np = ' ';
}
/* Note location of current '\0'. */
np = cp;
}
}
/*
* sp points to the beginning of the arguments/environment string, and
* np should point to the '\0' terminator for the string.
*/
if (np == NULL || np == sp) {
/* Empty or unterminated string. */
goto exit;
}
/* Make a copy of the string. */
process.commandLine = [NSString stringWithUTF8Strin:sp];
exit:
/* Clean up. */
free(procargs);
}
Would appreciate any help to understand what's can cause this kind of crash.
Look at the value of sp before you call this. I expect there's a bug in how you advance cp. You probably advance it too far. I would suspect problems in your = searching heuristic.
If sp is NULL, this will crash.
nullTerminatedCString
A NULL-terminated C array of bytes in UTF-8 encoding. This value must not be NULL.
Important
Raises an exception if nullTerminatedCString is NULL.

STL map allocation error in vc6

I'm using a map to merge duplicates and sort items in a dll. It's not exposed in the interface of the dll. The simplified code is as follows.
UINT GetInfo(UINT request, LPVOID data)
{
//...
switch (request)
{
case COUNT_RES:
{
CountRes* countRes = (CountRes*)data;
ZeroMemory(countRes, sizeof(CountRes));
try
{
//...
PUINT posValue = (PUINT)buffer;
PUINT posCount = (PUINT)buffer2;
FLOAT value; UINT count;
std::map<FLOAT, UINT, std::greater<FLOAT> > coinMap; //Access violation
countRes->rejected = ntohl(posCount[20]);
for (UCHAR i = 0; i < 20; ++i)
{
value = (FLOAT)ntohl(posValue[i]) / 100;
count = ntohl(posCount[i]);
coinMap[value] += count; //Access violation
countRes->total += value * count;
}
//...
}
//...
}
The GetInfo function is called from an exe. The showed code raises an access violation exception at the declaration line. The call stack is as follows.
_heap_alloc_dbg(unsigned int 0x00000018, int 0x00000001, const char * 0x00000000, int 0x00000000) line 394 + 8 bytes
_nh_malloc_dbg(unsigned int 0x00000018, int 0x00000001, int 0x00000001, const char * 0x00000000, int 0x00000000) line 242 + 21 bytes
_nh_malloc(unsigned int 0x00000018, int 0x00000001) line 194 + 19 bytes
operator new(unsigned int 0x00000018) line 24 + 11 bytes
std::_Allocate(int 0x00000018, char * 0x00000000) line 30 + 9 bytes
std::allocator<unsigned int>::_Charalloc(unsigned int 0x00000018) line 62 + 11 bytes
std::_Tree<float,std::pair<float const ,unsigned int>,std::map<float,unsigned int,std::greater<float>,std::allocator<unsigned int> >::_Kfn,std::greater<float>,std::allocator<unsigned int> >::_Buynode(...) line 587 + 10 bytes
std::_Tree<float,std::pair<float const ,unsigned int>,std::map<float,unsigned int,std::greater<float>,std::allocator<unsigned int> >::_Kfn,std::greater<float>,std::allocator<unsigned int> >::_Init() line 461 + 16 bytes
std::_Tree<float,std::pair<float const ,unsigned int>,std::map<float,unsigned int,std::greater<float>,std::allocator<unsigned int> >::_Kfn,std::greater<float>,std::allocator<unsigned int> >::_Tree<float,std::pair<float const ,unsigned int>,std::ma1aad805f(const std::greater<float> & {...}, unsigned char 0x00, const std::allocator<unsigned int> & {...}) line 162 + 67 bytes
std::map<float,unsigned int,std::greater<float>,std::allocator<unsigned int> >::map<float,unsigned int,std::greater<float>,std::allocator<unsigned int> >(const std::greater<float> & {...}, const std::allocator<unsigned int> & {...}) line 57 + 47 bytes
GetInfo(unsigned int 0xffffffff, void * 0x0012f658) line 331 + 25 bytes
If I declare coinMap out of the try scope, then the exception occurs at the insertion line instead. The call stack is as follows.
std::greater<float>::operator()(const float & 1.00000, const float &) line 80 + 37 bytes
std::_Tree<float,std::pair<float const ,unsigned int>,std::map<float,unsigned int,std::greater<float>,std::allocator<unsigned int> >::_Kfn,std::greater<float>,std::allocator<unsigned int> >::insert(const std::pair<float const ,unsigned int> & {...}) line 222 + 37 bytes
std::map<float,unsigned int,std::greater<float>,std::allocator<unsigned int> >::insert(const std::pair<float const ,unsigned int> & {...}) line 96 + 45 bytes
std::map<float,unsigned int,std::greater<float>,std::allocator<unsigned int> >::operator[](const float & 1.00000) line 93 + 65 bytes
GetInfo(unsigned int 0xffffffff, void * 0x0012f658) line 337 + 18 bytes
I have no clue how to solve the issue. Please help!
Crash in legal allocation code is 99.99% heap corruption issue.
Comment out all data / countRes references in GetInfo() and see is crash still raising. If yes very likely the problem is outside of code you posted here.

Arduino RFID checksum calculation and key visualization

I'm using this RFID module for Arduino Ethernet R3 and I need to retrieve from the Software Serial the card (TAG) ID that is written outside the tag.
The module's datasheet says that 14 bytes are sent to the Arduino. The first is the header, the last the footer, the 2 bytes before the footer are the checksum, and the other 10 bytes are the ASCII data that contains the tag ID.
How can I recreate the ID of the card, and control the checksum? For example with a tag that has this ID: 0013530444, the Arduino response is:
I received: 2
I received: 51
I received: 67
I received: 48
I received: 48
I received: 67
I received: 69
I received: 55
I received: 53
I received: 52
I received: 67
I received: 67
I received: 66
I received: 3
But I've no idea how to print on the screen the ID read by the Arduino. How to calculate the checksum?
http://www.seeedstudio.com/wiki/index.php?title=125Khz_RFID_module_-_UART
Can anyone help me?
Here's a walkthrough of how to calculate the checksum.
Take your card number (this is just directly quoted from your text)
I received: 2
I received: 51
I received: 67
I received: 48
I received: 48
I received: 67
I received: 69
I received: 55
I received: 53
I received: 52
I received: 67
I received: 67
I received: 66
I received: 3
This would give you a number that is equivalent to the following:
2 51 67 48 48 67 69 55 53 52 67 67 66 3
The first numer (2) indicates that this is the beginning of a request.
The last number (3) indicates that this is the end of a request.
2 51 67 48 48 67 69 55 53 52 67 67 66 3
For the purposes of calculating the checksum, we are going to remove these two numbers. So your new number is now:
51 67 48 48 67 69 55 53 52 67 67 66
The last two numbers that you have are your checksum. The remaining numbers are your card number. So:
Your card number is:
51 67 48 48 67 69 55 53 52 67
And your checksum is:
67 66
Next you need to convert your Card Number and your Checksum to ASCII values:
Your card number is:
3 C 0 0 C E 7 5 4 C
And your checksum is:
C B
Next, grab each number into pairs:
Your card number is:
3C 00 CE 75 4C
And your checksum is:
CB
Then you need to treat each pair as a HEXIDECIMAL value and do an XOR against them. So basically you need to prove the following:
3C ^ 00 ^ CE ^ 75 ^ 4C == CB
(3C ^ 00) = 3C
3C ^ CE ^ 75 ^ 4C == CB
(3C ^ CE) = F2
F2 ^ 75 ^ 4C == CB
(3C ^ CE) = 87
87 ^ 4C == CB
(87 ^ 4C) = CB
CB == CB
Because CB == CB, this is a valid transaction.
No doubt someone else can come up with a better approach than this, but there should be enough pseudo code here for you to write it yourself.
I found this blog which has an implementation in Arduino, I've adapted it to work in Java and results are good. Since there are a lot of bitwise operations - I used http://www.miniwebtool.com/bitwise-calculator/ to try to understand what's going on. I understand all of it except (val | (tempbyte << 4)), I mean I understand what the statement does, I just struggle to see how that produces the result I want.
void loop () {
byte i = 0;
byte val = 0;
byte code[6];
byte checksum = 0;
byte bytesread = 0;
byte tempbyte = 0;
if(Serial.available() > 0) {
if((val = Serial.read()) == 2) {
// check for header
bytesread = 0;
while (bytesread < 12) {
// read 10 digit code + 2 digit checksum
if( Serial.available() > 0) {
val = Serial.read();
if((val == 0x0D)||(val == 0x0A)||(val == 0x03)||(val == 0x02)) {
// if header or stop bytes before the 10 digit reading
break;
// stop reading
}
// Do Ascii/Hex conversion:
if ((val >= '0') && (val <= '9')) {
val = val - '0';
} else if ((val >= 'A') && (val <= 'F')) {
val = 10 + val - 'A';
}
// Every two hex-digits, add byte to code:
if (bytesread & 1 == 1) {
// make some space for this hex-digit by
// shifting the previous hex-digit with 4 bits to the left:
code[bytesread >> 1] = (val | (tempbyte << 4));
if (bytesread >> 1 != 5) {
// If we're at the checksum byte,
checksum ^= code[bytesread >> 1];
// Calculate the checksum... (XOR)
};
} else {
tempbyte = val;
// Store the first hex digit first...
};
bytesread++;
// ready to read next digit
}
}
// Output to Serial:
if (bytesread == 12) {
// removed code for clarity
LCD.print("Check:");
LCD.print(code[5], HEX);
LCD.print(code[5] == checksum ? "-passed" : "-error");
}
bytesread = 0;
}
}
}
My Java/Android port is listening over a BluetoothSocket. I'm using the code from BlueTerm as the base, this code goes in the ConnectedThread. Apologies for all silly comments, but I'm still learning Java).
//assume you have checksum as int and code as int array. it will overflow if bytes are used like above example
public void run() {
Log.d(TAG, "BEGIN mConnectedThread");
byte[] buffer = new byte[1024];
int bytes;
// Keep listening to the InputStream while connected
while (true) {
Log.d(TAG, "Running");
try {
// Read from the InputStream
bytes = mmInStream.read(buffer);
for (int i = 0; i < bytes; i++) {
Log.d(TAG, "Reading: " + i + " of " + bytes + " from input stream");
byte b = buffer[i];
try {
if(bytesread >= 0) {
//just printing ASCII
char printableB = (char) b;
if (b < 32 || b > 126) printableB = ' ';
Log.d(TAG, "'" + Character.toString(printableB) + "' (" + Integer.toString(b) + ")");
if((b == 0x0D)||(b == 0x0A)||(b == 0x03)||(b == 0x02)) {
// if header or stop bytes before the 10 digit reading
Log.e(TAG, i + " Unexpected header while processing character " + Integer.toString(b));
} else {
// Do ASCII/Hex conversion
if ((b >= '0') && (b <= '9')) {
b = (byte) (b - '0');
} else if ((b >= 'A') && (b <= 'F')) {
b = (byte) (10 + b - 'A');
}
if ((bytesread & 1) == 1) {
//if isOdd(bytesread)
// make some space for this hex-digit by shifting the previous hex-digit with 4 bits to the left:
code[bytesread >> 1] = (b | (tempbyte << 4));
if (bytesread >> 1 != 5) {
// If we're not at the checksum byte,
checksum ^= code[bytesread >> 1];
// Calculate the checksum... (XOR)
}
} else {
// Store the first hex digit first
tempbyte = b;
}
}
bytesread++;
} else if(b == 2) {
bytesread = 0;
Log.d(TAG, "Header found!");
}
if(bytesread == 12) {
String check = (code[5] == checksum ? "-passed" : "-error");
String r = "";
for(int j = 0; j < 5; j++){
r += Integer.toString(code[i]);
}
Log.d(TAG, "Check:" + Integer.toString(code[5]) + check);
init();
} else if(bytesread > 12){
Log.e(TAG, "Too many bytes!");
}
} catch (Exception e) {
Log.e(TAG, i + " Exception while processing character " + Integer.toString(b), e);
}
}
String a = buffer.toString();
a = "";
} catch (IOException e) {
Log.e(TAG, "disconnected", e);
connectionLost();
break;
}
}
Log.i(TAG, "END mConnectedThread");
}