I'd like to efficiently calculate a cumulative mode along an axis in numpy.
e.g.
>>> arr = np.random.RandomState(3).randint(3, size = (2, 5))
>>> arr
array([[2, 0, 1, 0, 0],
[0, 1, 1, 2, 1]])
>>> assert np.array_equal(cummode(arr, axis = 1), [[2,2,2,0,0],[0,0,1,1,1])
Is there an efficient way to do this? I guess it can handle ties by returning the first number to achieve the given count.
Here's a pure Python function that works on a list, or any iterable:
from collections import defaultdict
def cummode(alist):
dd = defaultdict(int)
mode = [(None,0)]
for i in alist:
dd[i] += 1
if dd[i]>mode[-1][1]:
newmode = (i,dd[i])
else:
newmode = mode[-1]
mode.append(newmode)
mode.pop(0)
return mode, dd
mode,dd = cummode([0,1,3,6,1,2,3,3,2,1,10,0])
print(dd)
print(mode)
which for the test case, produces
defaultdict(<type 'int'>, {0: 2, 1: 3, 2: 2, 3: 3, 6: 1, 10: 1})
[(0, 1), (0, 1), (0, 1), (0, 1), (1, 2), (1, 2), (1, 2), (3, 3), (3, 3), (3, 3), (3, 3), (3, 3)]
A detaultdict is a clean fast way of accumulating values when you don't know all the keys before hand. For small lists and arrays it probably beats a numpy based version, even with weave, simply because it does not incure the overhead of creating arrays. But with large ones it probably will lag. Plus I haven't generalized it to handle multiple values (rows).
Ok, it's not fully general in that it doesn't work along any axis, but for now I've made a version that works using scipy-weave.
from scipy import weave
def cummode(x, axis = 1):
assert x.ndim == 2 and axis == 1, 'Only implemented for a special case!'
all_values, element_ids = np.unique(x, return_inverse=True)
n_unique = len(all_values)
element_ids = element_ids.reshape(x.shape)
result = np.zeros(x.shape, dtype = int)
counts = np.zeros(n_unique, dtype = int)
code = """
int n_samples = Nelement_ids[0];
int n_events = Nelement_ids[1];
for (int i=0; i<n_samples; i++){
int maxcount = 0;
int maxel = -1;
for (int k=0; k<n_unique; k++)
counts[k] = 0;
for (int j=0; j<n_events; j++){
int ix = i*n_events+j;
int k = element_ids[ix];
counts[k]+=1;
if (counts[k] > maxcount){
maxcount = counts[k];
maxel = k;
}
result[ix]=maxel;
}
}
"""
weave.inline(code, ['element_ids', 'result', 'n_unique', 'counts'], compiler = 'gcc')
mode_values = all_values[result]
return mode_values
Related
I use the following function now, but I don't think it works, but I can't understand the description of the cupy kernel definition. This function is very memory intensive and time-consuming when it comes to huge data.
def cupy_sum(self, bins):
bidx = cupy.cumsum(bins) -1,
return cupy.diff(cupy.r_[0, cupy.cumsum(self)[bidx]])
Refer to other examples and write the following code, do not know if there is a problem.
sum_section_kernel = cp.ElementwiseKernel(
'raw T bins, raw T dats',
'float32 out',
'''
T bin_f = bins[i ];
T bin_l = bins[i+1];
T biv = 0;
for(size_t j=bin_f; j<bin_l; j++){
biv += dats[j];
}
out = biv;
''',
'summe')
a = cp.array([4, 3, 5], dtype=cp.float32)
b = cp.array([1, 1, 1.1, 1, 2, 2, 2, 3, 3, 3, 3, 3], dtype=cp.float32)
y = cp.empty(3, dtype=cp.float32)
a = cp.r_[0,a.cumsum()]
out = sum_section_kernel(a, b, y)
print(out)
> [ 4.100 6.000 15.000]
The example has been put in the above, and the speed has not been improved, but I think there is still the advantage of saving memory.
I'm trying to build an array of some given shape in which all elements are given by another array. Is there a function in numpy which does that efficiently, similar to np.full(), or any other elegant way, without simply employing for loops?
Example: Let's say I want an array with shape
(dim1,dim2) filled with a given, constant scalar value. Numpy has np.full() for this:
my_array = np.full((dim1,dim2),value)
I'm looking for an analog way of doing this, but I want the array to be filled with another array of shape (filldim1,filldim2) A brute-force way would be this:
my_array = np.array([])
for i in range(dim1):
for j in range(dim2):
my_array = np.append(my_array,fill_array)
my_array = my_array.reshape((dim1,dim2,filldim1,filldim2))
EDIT
I was being stupid, np.full() does take arrays as fill value if the shape is modified accordingly:
my_array = np.full((dim1,dim2,filldim1,filldim2),fill_array)
Thanks for pointing that out, #Arne!
You can use np.tile:
>>> shape = (2, 3)
>>> fill_shape = (4, 5)
>>> fill_arr = np.random.randn(*fill_shape)
>>> arr = np.tile(fill_arr, [*shape, 1, 1])
>>> arr.shape
(2, 3, 4, 5)
>>> np.all(arr[0, 0] == fill_arr)
True
Edit: better answer, as suggested by #Arne, directly using np.full:
>>> arr = np.full([*shape, *fill_shape], fill_arr)
>>> arr.shape
(2, 3, 4, 5)
>>> np.all(arr[0, 0] == fill_arr)
True
I have two multi-dimensional tensors a and b. And I want to sort them by the values of a.
I found tf.nn.top_k is able to sort a tensor and return the indices which is used to sort the input. How can I use the returned indices from tf.nn.top_k(a, k=2) to sort b?
For example,
import tensorflow as tf
a = tf.reshape(tf.range(30), (2, 5, 3))
b = tf.reshape(tf.range(210), (2, 5, 3, 7))
k = 2
sorted_a, indices = tf.nn.top_k(a, k)
# How to sort b into
# sorted_b[0, 0, 0, :] = b[0, 0, indices[0, 0, 0], :]
# sorted_b[0, 0, 1, :] = b[0, 0, indices[0, 0, 1], :]
# sorted_b[0, 1, 0, :] = b[0, 1, indices[0, 1, 0], :]
# ...
Update
Combining tf.gather_nd with tf.meshgrid can be one solution. For example, the following code is tested on python 3.5 with tensorflow 1.0.0-rc0:
a = tf.reshape(tf.range(30), (2, 5, 3))
b = tf.reshape(tf.range(210), (2, 5, 3, 7))
k = 2
sorted_a, indices = tf.nn.top_k(a, k)
shape_a = tf.shape(a)
auxiliary_indices = tf.meshgrid(*[tf.range(d) for d in (tf.unstack(shape_a[:(a.get_shape().ndims - 1)]) + [k])], indexing='ij')
sorted_b = tf.gather_nd(b, tf.stack(auxiliary_indices[:-1] + [indices], axis=-1))
However, I wonder if there is a solution which is more readable and doesn't need to create auxiliary_indices above.
Your code have a problem.
b = tf.reshape(tf.range(60), (2, 5, 3, 7))
Because TensorFlow Cannot reshape a tensor with 60 elements to shape [2,5,3,7] (210 elements).
And you can't sort a rank 4 tensor (b) using indices of rank 3 tensors.
I have an array like this:
arr = np.array([4, 6, 3, 9, 2, 100, 3, 1, 1, 1, 1])
I want to convert it to a bool array like this:
[ T, F, F, F, T, F, T, F, F, T, T]
# 4, 6, 3, 9, 2, 100, 3, 1, 1, 1, 1
I can do it with a loop like this:
mask = np.zeros(len(arr), dtype=bool)
ii = 0
while ii < len(arr):
mask[ii] = True
ii += arr[ii]
It's sort of an indirect indexing scheme, where each element in the input tells us how many subsequent elements are invalid.
How can I do it without using a Python loop, so that it will be fast if the input array is large? I'm happy to use Pandas too.
There may be some vectorization trick I'm not thinking of, but if you can use numba, it's well suited for problems like this - this loop should now be very fast.
import numba
#numba.jit(nopython=True)
def jump_mask(arr):
mask = np.zeros(len(arr), dtype=np.bool_)
ii = 0
while ii < len(arr):
mask[ii] = True
ii += arr[ii]
return mask
Let's say that I have an arbitrary number of iterables, all of which can be assumed to be sorted, and contain elements all of the same type (integers, for illustration's sake).
a = (1, 2, 3, 4, 5)
b = (2, 4, 5)
c = (1, 2, 3, 5)
I would like to write a generator function yielding the following:
(1, None, 1)
(2, 2, 2)
(3, None, 3)
(4, 4, None)
(5, 5, 5)
In other words, progressively yield sorted tuples with gaps where elements are missing from the input iterables.
My take on this, using only iterators, not heaps:
a = (1, 2, 4, 5)
b = (2, 5)
c = (1, 2, 6)
d = (1,)
inputs = [iter(x) for x in (a, b, c, d)]
def minwithreplacement(currents, inputs, minitem, done):
for i in xrange(len(currents)):
if currents[i] == minitem:
try:
currents[i] = inputs[i].next()
except StopIteration:
currents[i] = None
done[0] += 1
yield minitem
else:
yield None
def dothing(inputs):
currents = [it.next() for it in inputs]
done = [0]
while done[0] != len(currents):
yield minwithreplacement(currents, inputs, min(x for x in currents if x), done)
print [list(x) for x in dothing(inputs)] #Consuming iterators for display purposes
>>>[[1, None, 1, 1], [2, 2, 2, None], [4, None, None, None], [5, 5, None, None], [None, None, 6, None]]
We first need a variation of heapq.merge which also yields the index. You can get that by copy-pasting heapq.merge, and replacing each yield v with yield itnum, v. (I omit that part from my answer for readability).
Now we can do:
from collections import deque, OrderedDict
def f(*iterables):
pending = OrderedDict()
for i, v in merge(iterables):
if (not pending) or pending.keys()[-1] < v:
# a new greatest value
pending[v] = [None] * len(iterables)
pending[v][i] = v
# yield all values smaller than v
while len(pending) > 1 and pending.keys()[0] < v:
yield pending.pop(pending.keys()[0])
# yield remaining
while pending:
yield pending.pop(pending.keys()[0])
print list(f((1,2,3,4,5), (2,4,5), (1,2,3,5)))
=> [[1, None, 1], [2, 2, 2], [3, None, 3], [4, 4, None], [5, 5, 5]]