I'm trying to add units to a pandas DataFrame and it's containing Series, but if I slice it seems not to call init and I loose all the setup.
class QSeries(pd.Series):
_metadata = ["unit", "descriptor"]
#property
def _constructor(self):
return QSeries
#property
def _constructor_expanddim(self):
return QDataFrame
class QDataFrame(pd.DataFrame):
# normal properties
_metadata = ["units"]
def __init__(self, *args, units=None, **kwargs):
super(QDataFrame, self).__init__(*args, **kwargs)
if units is None:
self.units = {}
else:
self.units = {key:value for key, value in units.items() if key in self.columns}
for col in self.columns:
if col in self.units:
self[col].unit = self.units[col]
#property
def _constructor(self):
return QDataFrame
#property
def _constructor_sliced(self):
return QSeries
https://gist.github.com/wkerzendorf/52459080be83c7c382bac11ef9ac3195
test = QDataFrame(index=np.arange(10), columns=['x', 'y', 'z'], data=1., units={'x':u.erg, 'y':u.s, 'z':u.cm})
test.x.unit
output -> erg
test2 = test[['x', 'y']]
test2.x.unit
AttributeError
How do I fix this?
Related
I am trying to convert a pytorch script into tensorflow, how may I do so? Do we do it line by line or does the overall structure change in tensorflow?
Please, someone help me with this and provide some usefull link for this!
The code refers to graph convolution network. I see that pytorch_geometric has predefined modules like MessagePassing from which GCNConv is inheriting.
Is there any similar module in tensorflow?
GCN script :
import torch
from torch.nn import Parameter
from torch_scatter import scatter_add
from torch_geometric.nn import MessagePassing
from torch_geometric.utils import remove_self_loops, add_self_loops
from inits import glorot, zeros
import pdb
class GCNConv(MessagePassing):
def __init__(self,
in_channels,
out_channels,
improved=False,
cached=False,
bias=True):
super(GCNConv, self).__init__('add')
self.in_channels = in_channels
self.out_channels = out_channels
self.improved = improved
self.cached = cached
self.cached_result = None
self.weight = Parameter(torch.Tensor(in_channels, out_channels))
if bias:
self.bias = Parameter(torch.Tensor(out_channels))
else:
self.register_parameter('bias', None)
self.reset_parameters()
def reset_parameters(self):
glorot(self.weight)
zeros(self.bias)
self.cached_result = None
#staticmethod
def norm(edge_index, num_nodes, edge_weight, improved=False, dtype=None):
if edge_weight is None:
edge_weight = torch.ones((edge_index.size(1), ),
dtype=dtype,
device=edge_index.device)
edge_weight = edge_weight.view(-1)
assert edge_weight.size(0) == edge_index.size(1)
edge_index, edge_weight = remove_self_loops(edge_index, edge_weight)
edge_index, _ = add_self_loops(edge_index, num_nodes=num_nodes)
loop_weight = torch.full((num_nodes, ),
1 if not improved else 2,
dtype=edge_weight.dtype,
device=edge_weight.device)
edge_weight = torch.cat([edge_weight, loop_weight], dim=0)
row, col = edge_index
deg = scatter_add(edge_weight, col, dim=0, dim_size=num_nodes)
deg_inv_sqrt = deg.pow(-1)
deg_inv_sqrt[deg_inv_sqrt == float('inf')] = 0
return edge_index, deg_inv_sqrt[col] * edge_weight
def forward(self, x, edge_index, edge_weight=None):
""""""
x = torch.matmul(x, self.weight)
if not self.cached or self.cached_result is None:
edge_index, norm = self.norm(edge_index, x.size(0), edge_weight,
self.improved, x.dtype)
self.cached_result = edge_index, norm
edge_index, norm = self.cached_result
return self.propagate(edge_index, x=x, norm=norm)
def message(self, x_j, norm):
return norm.view(-1, 1) * x_j
def update(self, aggr_out):
if self.bias is not None:
aggr_out = aggr_out + self.bias
return aggr_out
def __repr__(self):
return '{}({}, {})'.format(self.__class__.__name__, self.in_channels,
self.out_channels)
The script is of a graph convolutional network. (source: https://github.com/seongjunyun/Graph_Transformer_Networks )
As a following of Astropy Units equivalencies - interferometry baselines. I would like to ask about how I can use SI prefixes for my custom-defined unit. Until now, and following the recommendations from the attached link, I made a subclass from astropy Quantity and then I have overridden the .to method.
Here is my code:
class uWavelength(un.Quantity):
def __new__(cls, value, freq=None, dtype=None, copy=True, **kwargs):
unit = un.Unit(un.def_unit('lambdas', format={'format': r'\lambda'}, prefixes=True))
self = super().__new__(cls, value=value, unit=unit, dtype=dtype, copy=copy, **kwargs)
self.freq = freq
if self.freq is not None:
self.equivalencies = self.lambdas_equivalencies()
return self
#property
def freq(self):
return self._freq
#freq.setter
def freq(self, val):
if val is not None:
self._equivalencies = self.lambdas_equivalencies(restfreq=val)
self._freq = val
#property
def equivalencies(self):
return self._equivalencies
#equivalencies.setter
def equivalencies(self, val):
self._equivalencies = val
def lambdas_equivalencies(self, restfreq=None):
if self.freq is not None:
restfreq_hz = self.freq.to(un.Hz, equivalencies=un.spectral())
elif restfreq is not None:
restfreq_hz = restfreq.to(un.Hz, equivalencies=un.spectral())
else:
sys.exit("Frequency not provided")
eq = [
(self.unit, un.s, lambda x: x / restfreq_hz, lambda x: x * restfreq_hz),
(self.unit, un.m, lambda x: x / restfreq_hz * co.c.to(un.m / un.s).value,
lambda x: x / co.c.to(un.m / un.s).value * restfreq_hz),
(un.m, un.s, lambda x: x / co.c.to(un.m / un.s).value, lambda x: x * co.c.to(un.m / un.s).value),
]
return eq
def to(self, unit, restfreq=None, copy=True):
equiv = []
if restfreq is None:
equiv = self.equivalencies
else:
equiv = self.lambdas_equivalencies(restfreq=restfreq)
unit = un.Unit(unit)
if copy:
# Avoid using to_value to ensure that we make a copy. We also
# don't want to slow down this method (esp. the scalar case).
value = self._to_value(unit, equiv)
else:
# to_value only copies if necessary
value = self.to_value(unit, equiv)
return self._new_view(value, unit)class uWavelength(un.Quantity):
def __new__(cls, value, freq=None, dtype=None, copy=True, **kwargs):
unit = un.Unit(un.def_unit('lambdas', format={'format': r'\lambda'}, prefixes=True))
self = super().__new__(cls, value=value, unit=unit, dtype=dtype, copy=copy, **kwargs)
self.freq = freq
if self.freq is not None:
self.equivalencies = self.lambdas_equivalencies()
return self
#property
def freq(self):
return self._freq
#freq.setter
def freq(self, val):
if val is not None:
self._equivalencies = self.lambdas_equivalencies(restfreq=val)
self._freq = val
#property
def equivalencies(self):
return self._equivalencies
#equivalencies.setter
def equivalencies(self, val):
self._equivalencies = val
def lambdas_equivalencies(self, restfreq=None):
if self.freq is not None:
restfreq_hz = self.freq.to(un.Hz, equivalencies=un.spectral())
elif restfreq is not None:
restfreq_hz = restfreq.to(un.Hz, equivalencies=un.spectral())
else:
sys.exit("Frequency not provided")
eq = [
(self.unit, un.s, lambda x: x / restfreq_hz, lambda x: x * restfreq_hz),
(self.unit, un.m, lambda x: x / restfreq_hz * co.c.to(un.m / un.s).value,
lambda x: x / co.c.to(un.m / un.s).value * restfreq_hz),
(un.m, un.s, lambda x: x / co.c.to(un.m / un.s).value, lambda x: x * co.c.to(un.m / un.s).value),
]
return eq
def to(self, unit, restfreq=None, copy=True):
equiv = []
if restfreq is None:
equiv = self.equivalencies
else:
equiv = self.lambdas_equivalencies(restfreq=restfreq)
unit = un.Unit(unit)
if copy:
# Avoid using to_value to ensure that we make a copy. We also
# don't want to slow down this method (esp. the scalar case).
value = self._to_value(unit, equiv)
else:
# to_value only copies if necessary
value = self.to_value(unit, equiv)
return self._new_view(value, unit)
However when I use the class I can only use the unit lambda, but I would like to use klambda or mega-lambda, etc. According to astropy this can be done by using the parameter prefixes=True, however this does not seem to work.
I don't think you should be actually defining the unit inside your class's __new__, as it's not allowing you to actually set the unit when you instantiate a uWavelength.
Instead put this outside your class:
lambdas = u.def_unit('lambdas', format={'format': r'\lambda'})
Something I think the docs don't make really clear is when you use prefixes=True it doesn't really do anything unless you also provide a namespace= argument, and even then the docs don't make it super clear how to use namespaces. I think it would be better to go with astrofrog's suggestion of explicitly declaring the prefixed units needed like:
klambdas = u.def_unit('kilolambdas', represents=u.CompositeUnit(1e3, [lambdas], [1]), format={'format' : r'k\lambda'})```
unless you *really* need every imaginable SI prefix, you could try:
lambdas = u.def_unit('lambdas', format={'format': r'\lambda'}, prefixes=True, namespace=globals())
and it will inject every prefixed unit into your module namespace.
Then, since you want your default unit for `uWavelength` to be `lambdas`, then both to reduce confusion, and also add some documentation of this fact (through the signature of `__new__`) specify:
```python
class uWavelength(un.Quantity):
def __new__(cls, value, freq=None, unit=lambdas, dtype=None, copy=True, **kwargs):
and further, if you wanted, you could add a check like:
unit = u.Unit(unit)
assert unit.is_equivalent(lambdas), 'unit must be equivalent to lambdas'
How do I create a combined iterator in MXNET? For example, given a record (.rec) iterator if I want to change the labels corresponding to each image then there are two options:
a) Create a new rec iterator with the same data(images) and new labels.
b) Create a multi-iterator using the original rec iterator and an NDArray iterator such that the multi-iterator reads data(images) from the original .rec iterator and labels from the NDArray iterator.
The option (a) is tedious. Any suggestions on how to create such a multi-iterator?
class MultiIter(mx.io.DataIter):
def __init__(self, iter_list):
self.iters = iter_list
self.batch_size = 1000
def next(self):
batches = [i.next() for i in self.iters]
return mx.io.DataBatch(data=[t for t in batches[0].data]+ [t for t in batches[1].data], label= [t for t in batches[0].label] + [t for t in batches[1].label],pad=0)
def reset(self):
for i in self.iters:
i.reset()
#property
def provide_data(self):
return [t for t in self.iters[0].provide_data] + [t for t in self.iters[1].provide_data]
#property
def provide_label(self):
return [t for t in self.iters[0].provide_label] + [t for t in self.iters[1].provide_label]
train = MultiIter([train1,train2])
Where train1 and train2 can be any two DataIter. In particular, train1 can be a .rec iterator and train2 can be an NDArray iterator. The additional argument "pad=0" is required for calling predict method using the combined iterator if either of train1 or train2 is an NDArray iterator.
MultiIter returns a list of data and a list of labels combined from the two iterators. If you need only data from the first iterator and labels from the second iterator, the code below will work.
class MultiIter(mx.io.DataIter):
def __init__(self, iter_list):
self.iters = iter_list
self.batch_size = 1000
def next(self):
batches = [i.next() for i in self.iters]
return mx.io.DataBatch(data=[t for t in batches[0].data], label= [t for t in batches[1].label],pad=0)
def reset(self):
for i in self.iters:
i.reset()
#property
def provide_data(self):
return [t for t in self.iters[0].provide_data]
#property
def provide_label(self):
return [t for t in self.iters[1].provide_label]
train = MultiIter([train1,train2])
I'm wonder wrt this topic
I want to resolve update issue in Theano.function with this lazy tensorflow constrution:
class TensorFlowTheanoFunction(object):
def __init__(self, inputs, outputs, session):
self._inputs = inputs
self._outputs = outputs
self.session = session
def __call__(self, *args, **kwargs):
feeds = {}
for (argpos, arg) in enumerate(args):
feeds[self._inputs[argpos]] = arg
return self.session.run(self._outputs, feeds)
If I want to pass an update argument (like in Theano) how I can modify this lazy call?
I just want that this can also work in tensorflow:
self.new = theano.function([], [], updates=zip(old_params, params))
Just modifying Yaroslav's code from that thread to use tf.assign, with a control dependency to make sure the outputs are computed before the assignments happen:
import tensorflow as tf
class TensorFlowTheanoFunction(object):
def __init__(self, inputs, outputs, updates=()):
self._inputs = inputs
self._outputs = outputs
self._updates = updates
def __call__(self, *args, **kwargs):
feeds = {}
for (argpos, arg) in enumerate(args):
feeds[self._inputs[argpos]] = arg
try:
outputs_identity = [tf.identity(output) for output in self._outputs]
output_is_list = True
except TypeError:
outputs_identity = [tf.identity(self._outputs)]
output_is_list = False
with tf.control_dependencies(outputs_identity):
assign_ops = [tf.assign(variable, replacement)
for variable, replacement in self._updates]
outputs_list = tf.get_default_session().run(
outputs_identity + assign_ops, feeds)[:len(outputs_identity)]
if output_is_list:
return outputs_list
else:
assert len(outputs_list) == 1
return outputs_list[0]
a = tf.placeholder(dtype=tf.int32)
b = tf.placeholder(dtype=tf.int32)
variable = tf.get_variable(
"variable", shape=[], dtype=tf.int32, initializer=tf.zeros_initializer)
c = a + b + variable
d = a - b
sess = tf.InteractiveSession()
sess.run(tf.initialize_all_variables())
f = TensorFlowTheanoFunction([a, b], [c, d], updates=[(variable, variable + 1)])
print f(1, 2)
print f(1, 2)
print f(0, 2)
f = TensorFlowTheanoFunction([a, b], c, updates=[(variable, variable + 1)])
print f(1, 2)
print f(1, 2)
print f(0, 2)
This updates the variable at each iteration:
[3, -1]
[4, -1]
[4, -2]
6
7
7
I'd like to set my rows to a fixed height. I've found an example using QAbstractItemModel, but I'm using QStandardItemModel. When I run the app, the QTreeView is blank. Any thoughts on how I could get this working for a QStandardItemModel?
import sys
from PySide import QtCore, QtGui
class TreeItem(object):
def __init__(self, data, parent=None):
self.parentItem = parent
self.data = data
self.childItems = []
def appendChild(self, item):
self.childItems.append(item)
def row(self):
if self.parentItem:
return self.parentItem.childItems.index(self)
return 0
class TreeModel(QtCore.QAbstractItemModel):
def __init__(self, parent=None):
super(TreeModel, self).__init__(parent)
self.rootItem = TreeItem(None)
for i, c in enumerate("abcdefg"):
child = TreeItem([i, c], self.rootItem)
self.rootItem.appendChild(child)
parent = self.rootItem.childItems[1]
child = TreeItem(["down", "down"], parent)
parent.appendChild(child)
def columnCount(self, parent):
return 2
def data(self, index, role):
if not index.isValid():
return None
if role == QtCore.Qt.DisplayRole:
item = index.internalPointer()
return item.data[index.column()]
elif role == QtCore.Qt.SizeHintRole:
print "giving size hint"
return QtCore.QSize(10, 10)
return None
def flags(self, index):
if not index.isValid():
return QtCore.Qt.NoItemFlags
return QtCore.Qt.ItemIsEnabled | QtCore.Qt.ItemIsSelectable
def headerData(self, section, orientation, role):
if orientation == QtCore.Qt.Horizontal and role == QtCore.Qt.DisplayRole:
return ["A", "B"][section]
return None
def index(self, row, column, parent):
if not self.hasIndex(row, column, parent):
return QtCore.QModelIndex()
if not parent.isValid():
parentItem = self.rootItem
else:
parentItem = parent.internalPointer()
childItem = parentItem.childItems[row]
if childItem:
return self.createIndex(row, column, childItem)
else:
return QtCore.QModelIndex()
def parent(self, index):
if not index.isValid():
return QtCore.QModelIndex()
parentItem = index.internalPointer().parentItem
if parentItem == self.rootItem:
return QtCore.QModelIndex()
return self.createIndex(parentItem.row(), 0, parentItem)
def rowCount(self, parent):
if parent.column() > 0:
return 0
if not parent.isValid():
parentItem = self.rootItem
else:
parentItem = parent.internalPointer()
return len(parentItem.childItems)
if __name__ == '__main__':
app = QtGui.QApplication(sys.argv)
model = TreeModel()
view = QtGui.QTreeView()
view.setModel(model)
view.setWindowTitle("Simple Tree Model")
view.show()
sys.exit(app.exec_())
Also, I've been looking around for the meaning of the term delegate in Qt, but the concept still isn't clicking in my head yet. Any insight into that would be appreciated as well!
You generally do this using QItemDelegates (or QStyledItemDelegate if you want stylesheet styling to work) by overriding the sizeHint method and always returning a size of a fixed height.
class MyDelegate(QtGui.QStyledItemDelegate):
def sizeHint(self, option, index):
my_fixed_height = 30
size = super(MyDelegate, self).sizeHint(option, index)
size.setHeight(my_fixed_height)
return size
view = QtGui.QTreeView()
delegate = MyDelegate()
view.setItemDelegate(delegate)