Related
The Arduino core is on github as ArduinoCore-mbed
I have had some success storing a camera buffer into Arduino MBED SDRAM, but that was fairly easy as the code was expecting a pointer when working with the camera. See code below:
SDRAMClass mySDRAM;
uint8_t *sdram_frame_buffer;
//uint8_t frame_buffer[320*320];
void setup() {
mySDRAM.begin();
sdram_frame_buffer = (uint8_t *)mySDRAM.malloc(320 * 320 * sizeof(uint8_t));
then in the main loop I did
if (cam.grab(sdram_frame_buffer) == 0){...
Note: I also do some frame alignment on the above code, but it still kind of works fine.
But now I want to store the entire TensorflowMicro c++ array
const unsigned char model_tflite[] = { 0x74, 0x69, 0x74, ...};
unsigned int model_tflite_len = 2640;
//which is later called as an array
model = tflite::GetModel(model_tflite); // name from the tflite converter model.h file
The problem here is that the Portenta has a max array size <= 1MB which is much less than the SDRAM max array size 8MB. What would be best would be to put the data for the Machine Learning model directly into SDRAM without using an array at all. Not sure if there is an easy way to do that.
Anyone have any suggestions?
Here is an example that runs on the Arduino Portenta
// Needed for SDRAM assignment on Arduino Portenta
#include <SDRAM.h>
#define ALIGN_PTR(p,a) ((p & (a-1)) ?(((uintptr_t)p + a) & ~(uintptr_t)(a-1)) : p)
SDRAMClass mySDRAM;
// assign values to a regular array
unsigned char model_tflite[] = {0x54, 0x46, 0x4c, 0x33};
unsigned int model_tflite_len = 4;
// define SDRAM pointer
unsigned char *sdram_mem;
unsigned char *sdram_tflite; // 32-byte aligned
void setup() {
Serial.begin(115200);
mySDRAM.begin(SDRAM_START_ADDRESS);
// setup SDRAM memory block
sdram_mem = (unsigned char *) SDRAM.malloc(4 + 32 /*alignment*/);
sdram_tflite = (unsigned char *)ALIGN_PTR((uintptr_t)sdram_mem, 32);
// THE PROBLEM
// How to assign the data directly to the allocated pointer memory
// without having to make an array as well
// The following traditional line works
// sdram_tflite = model_tflite;
// This line doesn't work
// sdram_tflite = {0x54, 0x46, 0x4c, 0x33};
// The following works, but is very clumsy
*(sdram_tflite + 0) = 0x54;
*(sdram_tflite + 1) = 0x46;
*(sdram_tflite + 2) = 0x4c;
*(sdram_tflite + 3) = 0x33;
}
void myShowArray( unsigned char b[], int sizeOfArray ) {
for ( int k = 0 ; k < sizeOfArray ; k++ ){
Serial.println(b[k], HEX);
}
Serial.println();
}
void myShowPointer( unsigned char *b, int sizeOfArray ) {
for ( int k = 0 ; k < sizeOfArray ; k++ ){
Serial.println(*(b + k), HEX);
}
Serial.println();
}
void loop() {
Serial.println("Regular array");
for (int i=0; i < model_tflite_len; i++){
Serial.println(model_tflite[i], HEX);
}
Serial.println();
Serial.println("SDRAM pointer as an array");
for (int i=0; i < model_tflite_len; i++){
Serial.println( *(sdram_tflite + i), HEX );
}
Serial.println();
Serial.println("Regular array passed as an array to the receiving function");
myShowArray(model_tflite, model_tflite_len);
Serial.println("Pointer passed as a pointer to the receiving function");
myShowPointer(sdram_tflite, model_tflite_len);
Serial.println("Pointer passed as an array to the receiving function");
myShowArray(*&sdram_tflite, model_tflite_len);
Serial.println("--------------------");
Serial.println();
delay(4000);
}
with output :
Regular array
54
46
4C
33
SDRAM pointer as an array
54
46
4C
33
Regular array passed as an array to the receiving function
54
46
4C
33
Pointer passed as a pointer to the receiving function
54
46
4C
33
Pointer passed as an array to the receiving function
54
46
4C
33
--------------------
// Needed for SDRAM assignment on Arduino Portenta
#include <SDRAM.h>
#define ALIGN_PTR(p,a) ((p & (a-1)) ?(((uintptr_t)p + a) & ~(uintptr_t)(a-1)) : p)
SDRAMClass mySDRAM;
// assign values to a regular array
unsigned char model_tflite[] = {0x54, 0x46, 0x4c, 0x33};
unsigned int model_tflite_len = 4;
// define SDRAM pointer
unsigned char *sdram_mem;
unsigned char *sdram_tflite; // 32-byte aligned
void setup() {
Serial.begin(115200);
mySDRAM.begin(SDRAM_START_ADDRESS);
// setup SDRAM memory block
sdram_mem = (unsigned char *) SDRAM.malloc(4 + 32 /*alignment*/);
sdram_tflite = (unsigned char *)ALIGN_PTR((uintptr_t)sdram_mem, 32);
// THE PROBLEM
// How to assign the data directly to the allocated pointer memory
// without having to make an array as well
// The following traditional line works
// sdram_tflite = model_tflite;
// This line doesn't work
// sdram_tflite = {0x54, 0x46, 0x4c, 0x33};
// The following works, but is very clumsy
// *(sdram_tflite + 0) = 0x54;
// *(sdram_tflite + 1) = 0x46;
// *(sdram_tflite + 2) = 0x4c;
// *(sdram_tflite + 3) = 0x33;
// Try this
// unsigned char buffer[8];
// memcpy(buffer,"\x20\x38\x00\x00\x0E\x82\x00\x06",8);
// yes this works fine!
memcpy(sdram_tflite, "\x54\x46\x4c\x33", model_tflite_len);
}
void myShowArray( unsigned char b[], int sizeOfArray ) {
for ( int k = 0 ; k < sizeOfArray ; k++ ){
Serial.println(b[k], HEX);
}
Serial.println();
}
void myShowPointer( unsigned char *b, int sizeOfArray ) {
for ( int k = 0 ; k < sizeOfArray ; k++ ){
Serial.println(*(b + k), HEX);
}
Serial.println();
}
void loop() {
Serial.println("Regular array");
for (int i=0; i < model_tflite_len; i++){
Serial.println(model_tflite[i], HEX);
}
Serial.println();
Serial.println("SDRAM pointer as an array");
for (int i=0; i < model_tflite_len; i++){
Serial.println( *(sdram_tflite + i), HEX );
}
Serial.println();
Serial.println("Regular array passed as an array to the receiving function");
myShowArray(model_tflite, model_tflite_len);
Serial.println("Pointer passed as a pointer to the receiving function");
myShowPointer(sdram_tflite, model_tflite_len);
Serial.println("Pointer passed as an array to the receiving function");
// myShowArray(*&sdram_tflite, model_tflite_len);
myShowArray(sdram_tflite, model_tflite_len);
Serial.println("--------------------");
Serial.println();
delay(4000);
}
I encountered the following error
gr::log :WARN: tpb_thread_body - asynchronous message buffer overflowing, dropping message
Out of serendipity, I ran into this GNU Radio presentation on
Youtube.
The presenter mentioned an OOT block he called "buffer" that is capable of eliminating the "buffer overflowing" error. Apparently, this block plays with different sample rates and the so-called "circular buffers". I haven't worked with circular buffers myself. Any ideas on circular buffers or any hints on how to build this buffer block are welcome.
EDIT
Below is the flowgraph that generates the error. As it was suggested in the comments, the culprits could be the message processing blocks (red-circled) namely generateCADU (for generating standard CCSDS frames) and processCADU (for extracting CADUs from a data stream).
The implementation file of the generateCADU block is given below
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <gnuradio/io_signature.h>
#include "generateCADU_impl.h"
#include "fec/ReedSolomon/ReedSolomon.h"
#include "fec/Scrambler/Scrambler.h"
namespace gr {
namespace ccsds {
generateCADU::sptr
generateCADU::make(int frameLength,std::string sync, int scramble, int rs, int intDepth)
{
return gnuradio::get_initial_sptr
(new generateCADU_impl(frameLength, sync, scramble, rs, intDepth));
}
/*
* The private constructor
*/
generateCADU_impl::generateCADU_impl(int frameLength,std::string sync, int scramble, int rs, int intDepth)
: gr::sync_block("generateCADU",
gr::io_signature::make(1, 1, sizeof(unsigned char)),
gr::io_signature::make(0, 0, 0)),
d_frameLength(frameLength),d_scramble(scramble == 1),d_rs(rs >= 1), d_basis(rs >= 2), d_intDepth(intDepth)
{
set_output_multiple(d_frameLength);
//Registering output port
message_port_register_out(pmt::mp("out"));
d_sync = parse_string(sync);
}
/*
* Our virtual destructor.
*/
generateCADU_impl::~generateCADU_impl()
{
}
unsigned char
generateCADU_impl::parse_hex(char c)
{
if ('0' <= c && c <= '9') return c - '0';
if ('A' <= c && c <= 'F') return c - 'A' + 10;
if ('a' <= c && c <= 'f') return c - 'a' + 10;
std::abort();
}
std::vector<unsigned char>
generateCADU_impl::parse_string(const std::string & s)
{
if (s.size() % 2 != 0) std::abort();
std::vector<unsigned char> result(s.size() / 2);
for (std::size_t i = 0; i != s.size() / 2; ++i)
result[i] = 16 * parse_hex(s[2 * i]) + parse_hex(s[2 * i + 1]);
return result;
}
int
generateCADU_impl::work(int noutput_items,
gr_vector_const_void_star &input_items,
gr_vector_void_star &output_items)
{
const unsigned char *in = (const unsigned char *) input_items[0];
//Reed-Solomon and Scrambler objects
ReedSolomon RS(16,d_intDepth,d_basis);// False = conventional, True = dual-basis
Scrambler S;
//Buffers
unsigned char *frameBuffer1 = (unsigned char*)malloc(d_frameLength*sizeof(unsigned char));
std::vector<unsigned char> frameBuffer2;
//The work function engine
for(int i = 0; (i + d_frameLength) < noutput_items; i += d_frameLength)
{
//Copying data from input stream
memcpy(frameBuffer1,in + i + d_frameLength,d_frameLength);
//Copying frame into std::vector buffer
frameBuffer2.insert(frameBuffer2.begin(),frameBuffer1, frameBuffer1 + d_frameLength);
//Optional scrambling and Reed-Solomon
if (d_rs) RS.Encode_RS(frameBuffer2);
if (d_scramble) S.Scramble(frameBuffer2);
//Insert sync word
frameBuffer2.insert(frameBuffer2.begin(), d_sync.begin(), d_sync.end());
//Transmitting PDU
pmt::pmt_t pdu(pmt::cons(pmt::PMT_NIL,pmt::make_blob(frameBuffer2.data(),frameBuffer2.size())));
message_port_pub(pmt::mp("out"), pdu);
//Clear buffer
frameBuffer2.clear();
}
// Tell runtime system how many output items we produced.
return noutput_items;
}
} /* namespace ccsds */
} /* namespace gr */
And here is the processCADU block. This block uses tags generated by the synchronizeCADU (which is simply a wrapper for the correlate_access_tag block) to extract CADUs
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <gnuradio/io_signature.h>
#include "processCADU_impl.h"
#include "fec/ReedSolomon/ReedSolomon.h"
#include "fec/Scrambler/Scrambler.h"
namespace gr {
namespace ccsds {
processCADU::sptr
processCADU::make(int frameLength, int scramble, int rs, int intDepth, std::string tagName)
{
return gnuradio::get_initial_sptr
(new processCADU_impl(frameLength, scramble, rs, intDepth, tagName));
}
/*
* The private constructor
*/
processCADU_impl::processCADU_impl(int frameLength, int scramble, int rs, int intDepth, std::string tagName)
: gr::sync_block("processCADU",
gr::io_signature::make(1, 1, sizeof(unsigned char)),
gr::io_signature::make(0, 0, 0)),
d_frameLength(frameLength),d_scramble(scramble == 1),d_rs(rs >= 1), d_basis(rs >= 2), d_intDepth(intDepth)
{
//Multiple input
set_output_multiple(d_frameLength * 8);
//Registering output port
message_port_register_out(pmt::mp("out"));
if (d_rs) d_frameLength += 32 * d_intDepth;
//SEtting tag name
key = pmt::mp(tagName);
}
/*
* Our virtual destructor.
*/
processCADU_impl::~processCADU_impl()
{
delete d_pack;
}
int
processCADU_impl::work(int noutput_items,
gr_vector_const_void_star &input_items,
gr_vector_void_star &output_items)
{
const unsigned char *in = (const unsigned char *) input_items[0];
unsigned char *out = (unsigned char *) output_items[0];
void *msg_data = NULL;
unsigned char frame_data[d_frameLength];
unsigned char frame_len = 0;
std::vector<unsigned char> frameBuffer;
//Reed-Solomon and Scrambler objects
ReedSolomon RS(16,d_intDepth,d_basis);// False = conventional, True = dual-basis
std::vector<int> errors;//errors.push_back(0);
Scrambler S;
d_tags.clear();
d_pack = new blocks::kernel::pack_k_bits(8);
this->get_tags_in_window(d_tags, 0, 0, noutput_items,key);
for(d_tags_itr = d_tags.begin(); d_tags_itr != d_tags.end(); d_tags_itr++) {
// Check that we have enough data for a full frame
if ((d_tags_itr->offset - this->nitems_read(0)) > (noutput_items - (d_frameLength) * 8))
{
return (d_tags_itr->offset - this->nitems_read(0) - 1);
}
//Pack bits into bytes
d_pack->pack(frame_data, &in[d_tags_itr->offset - this->nitems_read(0)], d_frameLength);
//Copying frame into std::vector buffer
frameBuffer.insert(frameBuffer.begin(),frame_data, frame_data + d_frameLength);
//Optional scrambling and Reed-Solomon
if (d_scramble) S.Scramble(frameBuffer);
//if (d_rs) RS.Decode_RS(frameBuffer,errors);
//If there is Reed-Solomon decoding
if(d_rs)
{
RS.Decode_RS(frameBuffer,errors);
if (RS.Success(errors)) // Success
{
//std::cout << "Success" << std::endl;
pmt::pmt_t pdu(pmt::cons(pmt::PMT_NIL,pmt::make_blob(frameBuffer.data(),frameBuffer.size())));
message_port_pub(pmt::mp("out"), pdu);
/*for(int i=0; i < errors.size(); i++)
{
//std::cout << "Number of Errors : " << errors.at(i) << std::endl << std::endl;
}*/
}
else // Failure
{
std::cout << "RS failure" << std::endl;
}
}
else{
pmt::pmt_t pdu(pmt::cons(pmt::PMT_NIL,pmt::make_blob(frameBuffer.data(),frameBuffer.size())));
message_port_pub(pmt::mp("out"), pdu);
}
//Clear buffers
frameBuffer.clear();
errors.clear();
}
// Tell runtime system how many output items we produced.
return noutput_items;
}
} /* namespace ccsds */
} /* namespace gr */
Regards,
M
Thanks to #MarcusMüller suggestion, using the tagged_stream paradigma as opposed to PDUs solved the problem. I was able to transmit 47 terabytes of data without any problems. Below is the code for the newly implemented block.
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <gnuradio/io_signature.h>
#include "genCADU_impl.h"
namespace gr {
namespace ccsds {
genCADU::sptr
genCADU::make(int frameLength,std::string sync, int scramble, int rs, int intDepth, std::string len_tag_key)
{
return gnuradio::get_initial_sptr
(new genCADU_impl(frameLength, sync, scramble, rs, intDepth, len_tag_key));
}
/*
* The private constructor
*/
genCADU_impl::genCADU_impl(int frameLength,std::string sync, int scramble, int rs, int intDepth, std::string len_tag_key)
: gr::tagged_stream_block("genCADU",
gr::io_signature::make(1, 1, sizeof(unsigned char)),
gr::io_signature::make(1, 1, sizeof(unsigned char)),len_tag_key),
d_frameLength(frameLength),d_scramble(scramble == 1),d_rs(rs >= 1), d_basis(rs >= 2), d_intDepth(intDepth)
{
//Synchronization pattern
d_sync = parse_string(sync);
//Reed-Solomon and Scrambler objects
RS = new ReedSolomon(16,d_intDepth,d_basis);// False = conventional, True = dual-basis
S = new Scrambler();
}
/*
* Our virtual destructor.
*/
genCADU_impl::~genCADU_impl()
{
delete RS;
delete S;
}
int
genCADU_impl::calculate_output_stream_length(const gr_vector_int &ninput_items)
{
int noutput_items = (d_rs) ? d_frameLength + 32*d_intDepth + d_sync.size() : d_frameLength + d_sync.size();
return noutput_items ;
}
unsigned char
genCADU_impl::parse_hex(char c)
{
if ('0' <= c && c <= '9') return c - '0';
if ('A' <= c && c <= 'F') return c - 'A' + 10;
if ('a' <= c && c <= 'f') return c - 'a' + 10;
std::abort();
}
std::vector<unsigned char>
genCADU_impl::parse_string(const std::string & s)
{
if (s.size() % 2 != 0) std::abort();
std::vector<unsigned char> result(s.size() / 2);
for (std::size_t i = 0; i != s.size() / 2; ++i)
result[i] = 16 * parse_hex(s[2 * i]) + parse_hex(s[2 * i + 1]);
return result;
}
int
genCADU_impl::work (int noutput_items,
gr_vector_int &ninput_items,
gr_vector_const_void_star &input_items,
gr_vector_void_star &output_items)
{
const unsigned char *in = (const unsigned char *) input_items[0];
unsigned char *out = (unsigned char *) output_items[0];
int total_len;
//Copy pdu from circular buffer to local buffer
buffer.insert(buffer.end(), in, in + d_frameLength);
//Optional scrambling and Reed-Solomon. TO DO: Turbo and LDPC
if (d_rs) RS->Encode_RS(buffer);
if (d_scramble) S->Scramble(buffer);
//Insert sync word
buffer.insert(buffer.begin(), d_sync.begin(), d_sync.end());
//Copy from local buffer to circular buffer
std::copy(buffer.begin(),buffer.end(),out);
//Clear the local buffer
total_len = buffer.size();
buffer.clear();
// Tell runtime system how many output items we produced.
return total_len;
}
} /* namespace ccsds */
} /* namespace gr */
Regards,
M.
I am currently conducting empirical studies to evaluate the run-time complexities of the quicksort, and mergesort algorithms. To do this I run a random number generator that stores whatever amount of numbers I specify in a binary file. The ranges of those numbers are from 1-1,000,000.I then run tests of each algorithm starting from 100,000 numbers, incrementing by 50,000 each time, until 1,000,000 numbers are sorted on the last run. So 20 tests each. I have successfully completed each algorithm but my results are kind of puzzingly. This is a graph showing my results.
I understand that quicksort has a worst case of O(n2) time, but typically O(n·lg(n)) time. Mergesort has Θ(n·lg(n)) time.
Also I would like to note that when I started the timer I just used clock() from time.h, and calculated the time elapsed. I started my timer one line of code before I called my sorting function.
What I dont understand is how my graph shows mergesort is always double the time, and reaching triple the time to sort numbers compared to quicksort.
My only thought is that for my mergesort algorithm every time I divide my array in half I use malloc to create a new integer array for each half. Of course this means a large amount of calls are made to malloc considering the number sizes I am sorting.
int* mergeSort(int* nums, int size){
int* left;
int* right;
int middle = size/2;
if(size <= 1)
return nums;
split(nums, size, &left, &right, middle);
//I dont understand why the code below wouldnt work in place of the split()
//when i run it, in main, nothing gets printed out. I guess i lose my pointer to the beginning of my array.
//left = nums;
//right = nums+middle;
left = mergeSort(left, middle);
right = mergeSort(right, size - middle);
merge(nums,left,right,middle,size - middle);
free(left);
free(right);
return nums;
}
void split(int* nums, int size, int** left, int** right, int middle){
int *lft = (int*) malloc ((sizeof(int) * middle));
int *rght = (int*) malloc ((sizeof(int) * size - middle));
int mid = middle;
int upMid = size - middle;
int i;
for(i=0; i < mid; i++)
lft[i] = nums[i];
for(i=0; i < upMid; i++)
rght[i] = nums[i+middle];
*left = lft;
*right = rght;
}
void merge(int* num, int* left, int* right, int sizeLeft, int sizeRight){
int i,j,k,n;
i=j=k=0;
n=sizeLeft + sizeRight;
while(k < n){
if(i< sizeLeft){
if(j<sizeRight){
insert(num,left,right,&i,&j,&k);
}
else{
append(num, left, sizeLeft, &i, &k);
}
}
else{
append(num,right,sizeRight,&j,&k);
}
}
}
void insert(int* num, int* left, int* right, int* i, int* j, int* k){
/*int i,j,k,n;*/
if(left[*i]<right[*j]){
num[*k] = left[*i];
(*i)++;
}
else{
num[*k] = right[*j];
(*j)++;
}
(*k)++;
}
void append(int* num, int* half, int sizeHalf, int* i, int* k){
while(*i < sizeHalf){
num[*k]= half[*i];
(*i)++; (*k)++;
}
}
I would greatly appreciate any feedback on this question of mine, and any advice on maybe making my mergesort function more efficient. Thanks!!
I have implemented a merge sort algorithm, you can have a look. I malloc a bak array at the beginning of mergeSort and every merge use the it afterwards.
#include <string>
#include <stdlib.h>
void _mergeSort(int *array, int *bakArray, int len) ;
void mergeSort(int *array, int len)
{
int *bak = (int *)malloc(sizeof(int)*len) ;
_mergeSort(array, bak, len) ;
free(bak) ;
}
void _mergeSort(int *array, int *bakArray, int len)
{
if (len >= 2) {
int leftLen = len/2 ;
_mergeSort(array, bakArray, leftLen) ;
_mergeSort(array+leftLen, bakArray, len-leftLen) ;
int *pa = array ;
int *pb = array+leftLen ;
int aIndex = 0 ;
int bIndex = 0 ;
while (aIndex < leftLen && bIndex < len-leftLen) {
int a = pa[aIndex] ;
int b = pb[bIndex] ;
if (a < b) {
bakArray[aIndex+bIndex] = a ;
++aIndex ;
} else if (a == b) {
bakArray[aIndex+bIndex] = a ;
bakArray[aIndex+bIndex+1] = a ;
++aIndex ;
++bIndex ;
} else {
bakArray[aIndex+bIndex] = b ;
++bIndex ;
}
}
if (aIndex < leftLen) {
memcpy(bakArray+aIndex+bIndex, pa+aIndex, sizeof(int)*(leftLen-aIndex)) ;
} else if (bIndex < len-leftLen) {
memcpy(bakArray+aIndex+bIndex, pb+bIndex, sizeof(int)*(len-leftLen-bIndex)) ;
}
memcpy(array, bakArray, sizeof(int)*len) ;
}
}
static const int MaxArraySize = 100 ;
int main()
{
srand(time(NULL)) ;
int array[MaxArraySize] ;
for (int i = 0 ; i < MaxArraySize; ++i) {
array[i] = rand() % 10000 ;
}
mergeSort(array, MaxArraySize) ;
for (int i = 0 ; i < MaxArraySize; ++i) {
printf("%d ", array[i]) ;
}
printf("\n") ;
return 0 ;
}
In my code i need to calculate CRC-16 16 bit values for the HEX values stored as NSdata, below is the code snippet to calculate CRC-16 in c.
void UpdateCRC(unsigned short int *CRC, unsigned char x)
{
// This function uses the initial CRC value passed in the first
// argument, then modifies it using the single character passed
// as the second argument, according to a CRC-16 polynomial
// Arguments:
// CRC -- pointer to starting CRC value
// x -- new character to be processed
// Returns:
// The function does not return any values, but updates the variable
// pointed to by CRC
static int const Poly = 0xA001;
int i;
bool flag;
*CRC ^= x;
for (i=0; i<8; i++)
// CRC-16 polynomial
{
flag = ((*CRC & 1) == 1);
*CRC = (unsigned short int)(*CRC >> 1);
if (flag)
*CRC ^= Poly;
}
return;
}
NSdata which holds the hex values like below
const char connectByteArray[] = {
0x21,0x01,0x90,0x80,0x5F
};
NSData* data = [NSData dataWithBytes: connectByteArray length:sizeof(connectByteArray)];
I solved using the following C program, I hope it may help someone ..cheers!!!
#include <string.h>
#include <stdio.h>
const int order = 16;
const unsigned long polynom = 0x8005;
const int direct = 1;
const unsigned long crcinit = 0;
const unsigned long crcxor = 0;
const int refin = 1;
const int refout = 1;
// 'order' [1..32] is the CRC polynom order, counted without the leading '1' bit
// 'polynom' is the CRC polynom without leading '1' bit
// 'direct' [0,1] specifies the kind of algorithm: 1=direct, no augmented zero bits
// 'crcinit' is the initial CRC value belonging to that algorithm
// 'crcxor' is the final XOR value
// 'refin' [0,1] specifies if a data byte is reflected before processing (UART) or not
// 'refout' [0,1] specifies if the CRC will be reflected before XOR
// Data character string
const unsigned char string[] = {0x05,0x0f,0x01,0x00,0x00,0x99};
// internal global values:
unsigned long crcmask;
unsigned long crchighbit;
unsigned long crcinit_direct;
unsigned long crcinit_nondirect;
unsigned long crctab[256];
// subroutines
unsigned long reflect (unsigned long crc, int bitnum) {
// reflects the lower 'bitnum' bits of 'crc'
unsigned long i, j=1, crcout=0;
for (i=(unsigned long)1<<(bitnum-1); i; i>>=1) {
if (crc & i) crcout|=j;
j<<= 1;
}
return (crcout);
}
void generate_crc_table() {
// make CRC lookup table used by table algorithms
int i, j;
unsigned long bit, crc;
for (i=0; i<256; i++) {
crc=(unsigned long)i;
if (refin) crc=reflect(crc, 8);
crc<<= order-8;
for (j=0; j<8; j++) {
bit = crc & crchighbit;
crc<<= 1;
if (bit) crc^= polynom;
}
if (refin) crc = reflect(crc, order);
crc&= crcmask;
crctab[i]= crc;
}
}
unsigned long crctablefast (unsigned char* p, unsigned long len) {
// fast lookup table algorithm without augmented zero bytes, e.g. used in pkzip.
// only usable with polynom orders of 8, 16, 24 or 32.
unsigned long crc = crcinit_direct;
if (refin) crc = reflect(crc, order);
if (!refin) while (len--) crc = (crc << 8) ^ crctab[ ((crc >> (order-8)) & 0xff) ^ *p++];
else while (len--) crc = (crc >> 8) ^ crctab[ (crc & 0xff) ^ *p++];
if (refout^refin) crc = reflect(crc, order);
crc^= crcxor;
crc&= crcmask;
return(crc);
}
unsigned long crctable (unsigned char* p, unsigned long len) {
// normal lookup table algorithm with augmented zero bytes.
// only usable with polynom orders of 8, 16, 24 or 32.
unsigned long crc = crcinit_nondirect;
if (refin) crc = reflect(crc, order);
if (!refin) while (len--) crc = ((crc << 8) | *p++) ^ crctab[ (crc >> (order-8)) & 0xff];
else while (len--) crc = ((crc >> 8) | (*p++ << (order-8))) ^ crctab[ crc & 0xff];
if (!refin) while (++len < order/8) crc = (crc << 8) ^ crctab[ (crc >> (order-8)) & 0xff];
else while (++len < order/8) crc = (crc >> 8) ^ crctab[crc & 0xff];
if (refout^refin) crc = reflect(crc, order);
crc^= crcxor;
crc&= crcmask;
return(crc);
}
unsigned long crcbitbybit(unsigned char* p, unsigned long len) {
// bit by bit algorithm with augmented zero bytes.
// does not use lookup table, suited for polynom orders between 1...32.
unsigned long i, j, c, bit;
unsigned long crc = crcinit_nondirect;
for (i=0; i<len; i++) {
c = (unsigned long)*p++;
if (refin) c = reflect(c, 8);
for (j=0x80; j; j>>=1) {
bit = crc & crchighbit;
crc<<= 1;
if (c & j) crc|= 1;
if (bit) crc^= polynom;
}
}
for (i=0; i<order; i++) {
bit = crc & crchighbit;
crc<<= 1;
if (bit) crc^= polynom;
}
if (refout) crc=reflect(crc, order);
crc^= crcxor;
crc&= crcmask;
return(crc);
}
unsigned long crcbitbybitfast(unsigned char* p, unsigned long len) {
// fast bit by bit algorithm without augmented zero bytes.
// does not use lookup table, suited for polynom orders between 1...32.
unsigned long i, j, c, bit;
unsigned long crc = crcinit_direct;
for (i=0; i<len; i++) {
c = (unsigned long)*p++;
if (refin) c = reflect(c, 8);
for (j=0x80; j; j>>=1) {
bit = crc & crchighbit;
crc<<= 1;
if (c & j) bit^= crchighbit;
if (bit) crc^= polynom;
}
}
if (refout) crc=reflect(crc, order);
crc^= crcxor;
crc&= crcmask;
return(crc);
}
int main() {
// test program for checking four different CRC computing types that are:
// crcbit(), crcbitfast(), crctable() and crctablefast(), see above.
// parameters are at the top of this program.
// Result will be printed on the console.
int i;
unsigned long bit, crc;
// at first, compute constant bit masks for whole CRC and CRC high bit
crcmask = ((((unsigned long)1<<(order-1))-1)<<1)|1;
crchighbit = (unsigned long)1<<(order-1);
// check parameters
if (order < 1 || order > 32) {
printf("ERROR, invalid order, it must be between 1..32.\n");
return(0);
}
if (polynom != (polynom & crcmask)) {
printf("ERROR, invalid polynom.\n");
return(0);
}
if (crcinit != (crcinit & crcmask)) {
printf("ERROR, invalid crcinit.\n");
return(0);
}
if (crcxor != (crcxor & crcmask)) {
printf("ERROR, invalid crcxor.\n");
return(0);
}
// generate lookup table
generate_crc_table();
// compute missing initial CRC value
if (!direct) {
crcinit_nondirect = crcinit;
crc = crcinit;
for (i=0; i<order; i++) {
bit = crc & crchighbit;
crc<<= 1;
if (bit) crc^= polynom;
}
crc&= crcmask;
crcinit_direct = crc;
}
else {
crcinit_direct = crcinit;
crc = crcinit;
for (i=0; i<order; i++) {
bit = crc & 1;
if (bit) crc^= polynom;
crc >>= 1;
if (bit) crc|= crchighbit;
}
crcinit_nondirect = crc;
}
// call CRC algorithms using the CRC parameters above and print result to the console
printf("\n");
printf("CRC tester v1.1 written on 13/01/2003 by Sven Reifegerste (zorc/reflex)\n");
printf("-----------------------------------------------------------------------\n");
printf("\n");
printf("Parameters:\n");
printf("\n");
printf(" polynom : 0x%x\n", polynom);
printf(" order : %d\n", order);
printf(" crcinit : 0x%x direct, 0x%x nondirect\n", crcinit_direct, crcinit_nondirect);
printf(" crcxor : 0x%x\n", crcxor);
printf(" refin : %d\n", refin);
printf(" refout : %d\n", refout);
printf("\n");
printf(" data string : '%s' (%d bytes)\n", string, strlen(string));
printf("\n");
printf("Results:\n");
printf("\n");
printf(" crc bit by bit : 0x%x\n", crcbitbybit((unsigned char *)string, 6));
printf(" crc bit by bit fast : 0x%x\n", crcbitbybitfast((unsigned char *)string, strlen(string)));
if (!(order&7)) printf(" crc table : 0x%x\n", crctable((unsigned char *)string, strlen(string)));
if (!(order&7)) printf(" crc table fast : 0x%x\n", crctablefast((unsigned char *)string, strlen(string)));
return(0);
}
I've looked through a lot of questions here for something similar and there are quite a few, albeit with one minor change. I'm trying to sort values with a zip_iterator as a compound key.
Specifically, I have the following function:
void thrustSort(
unsigned int * primaryKey,
float * secondaryKey,
unsigned int * values,
unsigned int numberOfPoints)
{
thrust::device_ptr dev_ptr_pkey = thrust::device_pointer_cast(primaryKey);
thrust::device_ptr dev_ptr_skey = thrust::device_pointer_cast(secondaryKey);
thrust::device_ptr dev_ptr_values = thrust::device_pointer_cast(values);
thrust::tuple,thrust::device_ptr> keytup_begin =
thrust::make_tuple,thrust::device_ptr>(dev_ptr_pkey, dev_ptr_skey);
thrust::zip_iterator, thrust::device_ptr > > first =
thrust::make_zip_iterator, thrust::device_ptr > >(keytup_begin);
thrust::sort_by_key(first, first + numberOfPoints, dev_ptr_values, ZipComparator());
}
and this custom predicate:
typedef thrust::device_ptr<unsigned int> tdp_uint ;
typedef thrust::device_ptr<float> tdp_float ;
typedef thrust::tuple<tdp_uint, tdp_float> tdp_uif_tuple ;
struct ZipComparator
{
__host__ __device__
inline bool operator() (const tdp_uif_tuple &a, const tdp_uif_tuple &b)
{
if(a.head < b.head) return true;
if(a.head == b.head) return a.tail < b.tail;
return false;
}
};
The errors I'm getting are:
Error 1 error : no instance of constructor "thrust::device_ptr::device_ptr [with T=unsigned int]" matches the argument list C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v4.0\include\thrust\detail\tuple.inl 309 1 ---
Error 2 error : no instance of constructor "thrust::device_ptr::device_ptr [with T=float]" matches the argument list C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v4.0\include\thrust\detail\tuple.inl 401 1 ---
Any ideas what might cause this / how do I write a predicate that indeed works?
Thanks in Advance,
Nathan
The comparator takes arguments of type const thrust::tuple<unsigned int, float>&. The const tdp_uif_tuple& type you defined expands to const thrust::tuple<thrust::device_ptr<unsigned int>, thrust:device_ptr<float> >&
The code below compiles for me:
struct ZipComparator
{
__host__ __device__
inline bool operator() (const thrust::tuple<unsigned int, float> &a, const thrust::tuple<unsigned int, float> &b)
{
if(a.head < b.head) return true;
if(a.head == b.head) return a.tail < b.tail;
return false;
}
};
Hope it does for you as well :)
http://code.google.com/p/thrust/wiki/QuickStartGuide#zip_iterator has more details on the zip iterator.
Not required, but if you're looking to clean up the length of those templates, you can do this:
void thrustSort(
unsigned int * primaryKey,
float * secondaryKey,
unsigned int * values,
unsigned int numberOfPoints)
{
tdp_uint dev_ptr_pkey(primaryKey);
tdp_float dev_ptr_skey(secondaryKey);
tdp_uint dev_ptr_values(values);
thrust::tuple<tdp_uint, tdp_float> keytup_begin = thrust::make_tuple(dev_ptr_pkey, dev_ptr_skey);
thrust::zip_iterator<thrust::tuple<tdp_uint, tdp_float> > first =
thrust::make_zip_iterator(keytup_begin);
thrust::sort_by_key(first, first + numberOfPoints, dev_ptr_values, ZipComparator());
}
A lot of the template arguments can be inferred from the arguments.
This is a fully worked example on how using sort_by_key when the key is a tuple dealt with by zip_iterator's and a customized comparison operator.
#include <thrust/device_vector.h>
#include <thrust/sort.h>
#include "Utilities.cuh"
// --- Defining tuple type
typedef thrust::tuple<int, int> Tuple;
/**************************/
/* TUPLE ORDERING FUNCTOR */
/**************************/
struct TupleComp
{
__host__ __device__ bool operator()(const Tuple& t1, const Tuple& t2)
{
if (t1.get<0>() < t2.get<0>())
return true;
if (t1.get<0>() > t2.get<0>())
return false;
return t1.get<1>() < t2.get<1>();
}
};
/********/
/* MAIN */
/********/
int main()
{
const int N = 8;
// --- Keys and values on the host: allocation and definition
int h_keys1[N] = { 1, 3, 3, 3, 2, 3, 2, 1 };
int h_keys2[N] = { 1, 5, 3, 8, 2, 8, 1, 1 };
float h_values[N] = { 0.3, 5.1, 3.2, -0.08, 2.1, 5.2, 1.1, 0.01};
printf("\n\n");
printf("Original\n");
for (int i = 0; i < N; i++) {
printf("%i %i %f\n", h_keys1[i], h_keys2[i], h_values[i]);
}
// --- Keys and values on the device: allocation
int *d_keys1; gpuErrchk(cudaMalloc(&d_keys1, N * sizeof(int)));
int *d_keys2; gpuErrchk(cudaMalloc(&d_keys2, N * sizeof(int)));
float *d_values; gpuErrchk(cudaMalloc(&d_values, N * sizeof(float)));
// --- Keys and values: host -> device
gpuErrchk(cudaMemcpy(d_keys1, h_keys1, N * sizeof(int), cudaMemcpyHostToDevice));
gpuErrchk(cudaMemcpy(d_keys2, h_keys2, N * sizeof(int), cudaMemcpyHostToDevice));
gpuErrchk(cudaMemcpy(d_values, h_values, N * sizeof(float), cudaMemcpyHostToDevice));
// --- From raw pointers to device_ptr
thrust::device_ptr<int> dev_ptr_keys1 = thrust::device_pointer_cast(d_keys1);
thrust::device_ptr<int> dev_ptr_keys2 = thrust::device_pointer_cast(d_keys2);
thrust::device_ptr<float> dev_ptr_values = thrust::device_pointer_cast(d_values);
// --- Declare outputs
thrust::device_vector<float> d_values_output(N);
thrust::device_vector<Tuple> d_keys_output(N);
auto begin_keys = thrust::make_zip_iterator(thrust::make_tuple(dev_ptr_keys1, dev_ptr_keys2));
auto end_keys = thrust::make_zip_iterator(thrust::make_tuple(dev_ptr_keys1 + N, dev_ptr_keys2 + N));
thrust::sort_by_key(begin_keys, end_keys, dev_ptr_values, TupleComp());
int *h_keys1_output = (int *)malloc(N * sizeof(int));
int *h_keys2_output = (int *)malloc(N * sizeof(int));
float *h_values_output = (float *)malloc(N * sizeof(float));
gpuErrchk(cudaMemcpy(h_keys1_output, d_keys1, N * sizeof(int), cudaMemcpyDeviceToHost));
gpuErrchk(cudaMemcpy(h_keys2_output, d_keys2, N * sizeof(int), cudaMemcpyDeviceToHost));
gpuErrchk(cudaMemcpy(h_values_output, d_values, N * sizeof(float), cudaMemcpyDeviceToHost));
printf("\n\n");
printf("Ordered\n");
for (int i = 0; i < N; i++) {
printf("%i %i %f\n", h_keys1_output[i], h_keys2_output[i], h_values_output[i]);
}
}