Numpy fails to load via JNI - numpy

I am trying to call python from a JNI library. This works as expected and thanks to all the information around on the interweb.
My problem begins when I try to "import numpy as np". When I include that in any python script, PyImport_Import(pName) in the code below returns null. Without the check for null, the program (expectedly) issues a segmentation violation.
I am using Java 7 (openjdk-amd64), Ubuntu 13.04, gcc 4.7.3
The code below is a snippet of the python interpreter initialisation and call. It is compiled into a shared object loaded by Java. If the same code is included in a normal program, the python script runs exactly as expected.
My guess is that numpy is unable to load libraries it requires when run via java and the shared JNI library. But after 48 hours of work, I'm clueless as to what I need to say to it to make it work.
int call_python(char *module, char *function, PyObject *list)
{
PyObject *pName, *pModule, *pDict, *pFunc, *pValue;
Py_Initialize();
printf("Initialized\n");
pName = PyString_FromString(module);
printf("pName created from: '%s'\n", module);
PyRun_SimpleString("import sys");
PyRun_SimpleString("sys.path.append(\".\")");
PyRun_SimpleString("sys.path.append(\"/home/jonathan/swigtests/j2p\")");
PyRun_SimpleString("print \"Interpreter Running\"");
pModule = PyImport_Import(pName);
if (pModule == (PyObject *)NULL)
{
printf("Unable to load '%s'\n", module);
return 0;
}
printf("Checking pmodule...%x\n", (int)pModule);
if (!PyModule_Check(pModule))
printf("pModule is not a module!\n");
else
printf("pModule is a module!\n");
pDict = PyModule_GetDict(pModule);
pFunc = PyDict_GetItemString(pDict, function);
PyObject *arglist = Py_BuildValue("(i)", 1);
PyDict_SetItemString(pDict, "field", list);
if (PyCallable_Check(pFunc))
{
printf("Calling Python\n");
//if (PyObject_CallFunction(pFunc, "i", 2) == NULL)
if (PyObject_CallObject(pFunc, arglist) == NULL)
printf("CallFunction failed\n");
else
printf("CallFunction succeeded\n");
}
else
{
printf("Not callable\n");
PyErr_Print();
}
printf("Python done\n");
Py_DECREF(pModule);
Py_DECREF(pName);
Py_Finalize();
return 0;
}
Update. I added PyErr_Print after printf("Unable to load")... This yields the traceback printout that is helpful:
Traceback (most recent call last):
File "./py_function.py", line 3, in <module>
import numpy as np
File "/usr/lib/python2.7/dist-packages/numpy/__init__.py", line 137, in <module>
import add_newdocs
File "/usr/lib/python2.7/dist-packages/numpy/add_newdocs.py", line 9, in <module>
from numpy.lib import add_newdoc
File "/usr/lib/python2.7/dist-packages/numpy/lib/__init__.py", line 4, in <module>
from type_check import *
File "/usr/lib/python2.7/dist-packages/numpy/lib/type_check.py", line 8, in <module>
import numpy.core.numeric as _nx
File "/usr/lib/python2.7/dist-packages/numpy/core/__init__.py", line 5, in <module>
import multiarray
ImportError: /usr/lib/python2.7/dist-packages/numpy/core/multiarray.so: undefined symbol: PyExc_SystemError
This starts to look like another problem reported with DateTime:
https://mail.python.org/pipermail//capi-sig/2013-August/000600.html
Exactly the same as this problem:
How to allow 3rd-party libraries to be imported by scripts called in embedded Python interpreter?

Related

Sqlfluff: Cannot instantiate a templated file unsliced

I recently decided to add sqlfluff to my dbt project so i just followed https://docs.sqlfluff.com/en/stable/production.html and added sqlfluff with pre-commit to my project.
Following is how my very basic .sqlfluff configurations look like:
[sqlfluff]
dialect = postgres
templater = jinja
output_line_length = 80
ignore_templated_areas = True
runaway_limit = 100
[sqlfluff:rules]
tab_space_size = 2
max_line_length = 120
indent_unit = space
comma_style = trailing
[sqlfluff:rules:L014]
extended_capitalisation_policy = lower
[sqlfluff:templater:jinja]
apply_dbt_builtins = true
Here is .pre-commit-config.yaml:
repos:
- repo: https://github.com/sqlfluff/sqlfluff
rev: 1.0.0
hooks:
- id: sqlfluff-lint
name: sqlfluff-lint
entry: sqlfluff lint
language: python
description: 'Lints sql files with `SQLFluff`'
types: [sql]
require_serial: true
additional_dependencies: []
- id: sqlfluff-fix
name: sqlfluff-fix
# Needs to use "--force" to disable confirmation
# By default all the rules are applied
entry: sqlfluff fix --force
language: python
description: 'Fixes sql lint errors with `SQLFluff`'
types: [sql]
require_serial: true
additional_dependencies: []
Once I run pre-commit run --all-files I end up getting the following:
File "/root/.cache/pre-commit/repo20y2aa42/py_env-python3.8/bin/sqlfluff", line 8, in <module>
sys.exit(cli())
File "/root/.cache/pre-commit/repo20y2aa42/py_env-python3.8/lib/python3.8/site-packages/click/core.py", line 1130, in __call__
return self.main(*args, **kwargs)
File "/root/.cache/pre-commit/repo20y2aa42/py_env-python3.8/lib/python3.8/site-packages/click/core.py", line 1055, in main
rv = self.invoke(ctx)
File "/root/.cache/pre-commit/repo20y2aa42/py_env-python3.8/lib/python3.8/site-packages/click/core.py", line 1657, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/root/.cache/pre-commit/repo20y2aa42/py_env-python3.8/lib/python3.8/site-packages/click/core.py", line 1404, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/root/.cache/pre-commit/repo20y2aa42/py_env-python3.8/lib/python3.8/site-packages/click/core.py", line 760, in invoke
return __callback(*args, **kwargs)
File "/root/.cache/pre-commit/repo20y2aa42/py_env-python3.8/lib/python3.8/site-packages/sqlfluff/cli/commands.py", line 769, in fix
result = lnt.lint_paths(
File "/root/.cache/pre-commit/repo20y2aa42/py_env-python3.8/lib/python3.8/site-packages/sqlfluff/core/linter/linter.py", line 1143, in lint_paths
self.lint_path(
File "/root/.cache/pre-commit/repo20y2aa42/py_env-python3.8/lib/python3.8/site-packages/sqlfluff/core/linter/linter.py", line 1095, in lint_path
for i, linted_file in enumerate(runner.run(fnames, fix), start=1):
File "/root/.cache/pre-commit/repo20y2aa42/py_env-python3.8/lib/python3.8/site-packages/sqlfluff/core/linter/runner.py", line 101, in run
for fname, partial in self.iter_partials(fnames, fix=fix):
File "/root/.cache/pre-commit/repo20y2aa42/py_env-python3.8/lib/python3.8/site-packages/sqlfluff/core/linter/runner.py", line 54, in iter_partials
for fname, rendered in self.iter_rendered(fnames):
File "/root/.cache/pre-commit/repo20y2aa42/py_env-python3.8/lib/python3.8/site-packages/sqlfluff/core/linter/runner.py", line 43, in iter_rendered
yield fname, self.linter.render_file(fname, self.config)
File "/root/.cache/pre-commit/repo20y2aa42/py_env-python3.8/lib/python3.8/site-packages/sqlfluff/core/linter/linter.py", line 816, in render_file
return self.render_string(raw_file, fname, config, encoding)
File "/root/.cache/pre-commit/repo20y2aa42/py_env-python3.8/lib/python3.8/site-packages/sqlfluff/core/linter/linter.py", line 787, in render_string
templated_file, templater_violations = self.templater.process(
File "/root/.cache/pre-commit/repo20y2aa42/py_env-python3.8/lib/python3.8/site-packages/sqlfluff/core/templaters/jinja.py", line 413, in process
TemplatedFile(
File "/root/.cache/pre-commit/repo20y2aa42/py_env-python3.8/lib/python3.8/site-packages/sqlfluff/core/templaters/base.py", line 102, in __init__
raise ValueError("Cannot instantiate a templated file unsliced!")
ValueError: Cannot instantiate a templated file unsliced!```
Not sure what breaking this
so first off your configuration has a bunch of unnecessary things in it -- let's pare that down (you also don't need both -lint and -fix -- they're redundant):
repos:
- repo: https://github.com/sqlfluff/sqlfluff
rev: 1.0.0
hooks:
- id: sqlfluff-fix
and with that configuration I can't reproduce your error -- you'll need to provide a sql file which triggers the particular error I suspect and your associated templates
disclaimer: I created pre-commit
I had the same issue, and making sure to exclude folders (macros and dbt_packages) having templated files solved this issue for me.
https://docs.sqlfluff.com/en/stable/configuration.html
Print screen of default .sqlfluffignore file on official site:
.sqlfluffignore default file
I hope this helps!

Python3/Redis: redis.exceptions.ResponseError: unknown command 'JSON.SET'

I'm trying to run the sample program from this RedisLabs page.
I chose Option A - which was to set up the free Redis cloud server.
(Seems like if you install manually, then you have to add the JSON as a plugin.)
I'm able to connect and use other "set" commands, but getting error on JSON:
File "C:\Users\nwalt\.virtualenvs\TDAmeritradeGetQuotes\lib\site-packages\redis\client.py", line 901, in execute_command
return self.parse_response(conn, command_name, **options)
File "C:\Users\nwalt\.virtualenvs\TDAmeritradeGetQuotes\lib\site-packages\redis\client.py", line 915, in parse_response
response = connection.read_response()
File "C:\Users\nwalt\.virtualenvs\TDAmeritradeGetQuotes\lib\site-packages\redis\connection.py", line 756, in read_response
raise response
redis.exceptions.ResponseError: unknown command 'JSON.SET'
My Python test program (except put in the sample endpoint before posting):
import redis
import json
import pprint
host_info = "redis.us-east-1-1.ec2.cloud.redislabs.com"
redisObj = redis.Redis(host=host_info, port=18274, password='xxx')
print ("Normal call to Redis")
redisObj.set('foo', 'bar')
value = redisObj.get('foo')
print(value)
capitals = {
"Lebanon": "Beirut",
"Norway": "Oslo",
"France": "Paris"
}
print ("capitals - before call to Redis")
pprint.pprint(capitals)
print("JSON call to Redis")
redisObj.execute_command('JSON.SET', 'doc', '.', json.dumps(capitals))
print("Data Saved, now fetch data back from redis")
reply = json.loads(redisObj.execute_command('JSON.GET', 'doc'))
print("reply from Redis get")
pprint.pprint(reply)
This is the screen shot from their website where I created the database. I didn't see any option to enable JSON or add any modules.
Not sure this was available when I created the REDIS database, but it is now. When you create it on redislabs.com, you can turn on the modules, and pick one from the list.
Then use this library: "rejson" from https://pypi.org/project/rejson/ to get the method "jsonset" method, using such code such as this:
rj = Client(host=config_dict['REDIS_CONFIG_HOST'], port=config_dict['REDIS_CONFIG_PORT'], password=config_dict['REDIS_CONFIG_PASSWORD'], decode_responses=True)
out_doc = {}
out_doc['firstname'] = "John"
out_doc['lastname'] = "Doe"
rj.jsonset('config', Path.rootPath(), out_doc)
get_doc = rj.jsonget('config', Path.rootPath())
pprint.pprint(get_doc)
I'm not used the cloud redis, in my local the Python don't load the JSON.SET
I just so make done, in this sample https://onelinerhub.com/python-redis/save-json-to-redis

Aerospike: zlib/bz2 store and retrieve didnt worked

I am compressing a string using zlib, then storing in Aerospike bin. On retrieval and decompressing, I am getting "zlib.error: Error -5 while decompressing data: incomplete or truncated stream"
When I compared original compressed data and retrieved compressed data, some thing is missing at the end in retrieved data.
I am using Aerospike 3.7.3 & python client 2.0.1
Please help
Thanks
Update: Tried using bz2. Throws ValueError: couldn't find end of stream while retrieve and decompress. Looks like aerospike is stripping of the last byte or something else from the blob.
Update: Posting the code
import aerospike
import bz2
config = {
'hosts': [
( '127.0.0.1', 3000 )
],
'policies': {
'timeout': 1000 # milliseconds
}
}
client = aerospike.client(config)
client.connect()
content = "An Aerospike Query"
content_bz2 = bz2.compress(content)
key = ('benchmark', 'myset', 55)
#client.put(key, {'bin0':content_bz2})
(key, meta, bins) = client.get(key)
print bz2.decompress(bins['bin0'])
Getting Following Error:
Traceback (most recent call last):
File "asread.py", line 22, in <module>
print bz2.decompress(bins['bin0'])
ValueError: couldn't find end of stream
The bz.compress method returns a string, and the client sees that type and tries to convert it to the server's as_str type. If it runs into a \0 in an unexpected position it will truncate the string, causing your error.
Instead, make sure to cast binary data to a bytearray, which the client converts to the server's as_bytes type. On the read operation, bz.decompress will work with the bytearray data and give you back the original string.
from __future__ import print_function
import aerospike
import bz2
config = {'hosts': [( '33.33.33.91', 3000 )]}
client = aerospike.client(config)
client.connect()
content = "An Aerospike Query"
content_bz2 = bytearray(bz2.compress(content))
key = ('test', 'bytesss', 1)
client.put(key, {'bin0':content_bz2})
(key, meta, bins) = client.get(key)
print(type(bins['bin0']))
bin0 = bz2.decompress(bins['bin0'])
print(type(bin0))
print(bin0)
Gives back
<type 'bytearray'>
<type 'str'>
An Aerospike Query

ComplexModel not available on the client

I just started using Spyne and tried to use a ComplexModel as a parameter for one method. I mostly followed the user_manager example from the sources with spyne<2.99 but I always get a type error when doing the client.factory.create() call.
Example code that fails:
from spyne.application import Application
from spyne.decorator import rpc
from spyne.service import ServiceBase
from spyne.protocol.soap import Soap11
from spyne.model.primitive import String, Integer
from spyne.model.complex import ComplexModel
class DatosFac(ComplexModel):
__namespace__ = 'facturamanager.datosfac'
numero = String(pattern=r'[A-Z]/[0-9]+')
class FacturaService(ServiceBase):
#rpc(String, DatosFac, _returns=Integer)
def updateFacData(self, numero, data):
# do stuff
return 1
application = Application([FacturaService], 'facturaManager.service',
in_protocol=Soap11(validator='lxml'),
out_protocol=Soap11()
)
from spyne.server.null import NullServer
s = NullServer(application)
data = s.factory.create('DatosFac')
If you run this code you get:
Traceback (most recent call last):
File "spyner.py", line 25, in <module>
data = s.factory.create('DatosFac')
File "/Users/marc/.pyEnvs/default/lib/python2.7/site-packages/spyne/client/_base.py", line 30, in create
return self.__app.interface.get_class_instance(object_name)
File "/Users/marc/.pyEnvs/default/lib/python2.7/site-packages/spyne/interface/_base.py", line 114, in get_class_instance
return self.classes[key]()
KeyError: 'DatosFac'
(I used NullServer to make it easier to reproduce, but the same happens over Soap+Wsgi).
I amb pretty much stuck at this as I don't see what's essentialy different from this code and the user_manager examples.
What am I doing wrong?
thanks,
marc
Thanks for providing a fully working example.
The difference is that tns and the namespace of the DatosFac are different.
Either do:
data = s.factory.create('{facturamanager.datosfac}DatosFac')
or remove __namespace__ from DatosFac definition

What is the best way to get a content object's position in the parent? Plone 4

I am traversing folders with content items within them. I use the portal_catalog to get brains searched on certain paths. The brains have access to metadata, and brain.getObject() will return the actual object. I have gotten the parent for an object with brain.getObject().aq_parent. Now I want to get the object's position in the parent. At first I tried brain.getObject().getObjPositionInParent(), and afterwards, I realized that the getObjPositionInParent() is an attribute accessible from the index data.
idxData = catalog.getIndexDataForRID(brain.getRID())
sJson = json.dumps( idxData )
l = brain.getObject()
lUpdate = {'path': '/'.join( l.getPhysicalPath()), 'meta_type': l.meta_type, 'title':l.getRawTitle(), 'remoteUrl': l.getRemoteUrl(), 'json':sJson}
When I printed this out to the screen, I see all of the items within the dict that is returned from the catalog.getIndexDataForRID call. The problem is that for all of the objects, getObjPositionInParent() is an empty array ([]). On this page http://developer.plone.org/searching_and_indexing/query.html, it appears the value should be an integer. This made me wonder if I have to go create the index data, and, if so, then I might be reaching too far outward from the object to get data that must already be there (since the folders obviously know what position to put each child in). What is the best way to get a content object's position in the parent? Thanks in advance for any information?
More:
I am unsure why the adapter cannot be found, but it may have to do with lack of registering it. This is a script that I build the Zope environment to read the ZODB directly from the file as opposed to on top of the running Zope instance. Is it possible that I have to register the adapter with the GlobalSiteManager?
Thank you, Mathias. When I use the sort_on="getObjPositionInParent", I get the following error:
Traceback (most recent call last):
File "extractMenuStructure.py", line 459, in <module>
res = processFolder( home['childItems'], '/Sanford Guide Web Edition/' + appFolderNm + '', config['screens'] )
File "extractMenuStructure.py", line 390, in processFolder
results = portal_catalog(path={"query":currentPath, "depth":d},sort_on="getObjPositionInParent")
File "/Applications/Plone/buildout-cache/eggs/Products.CMFPlone-4.1.2-py2.6.egg/Products/CMFPlone/CatalogTool.py", line 427, in searchResults
return ZCatalog.searchResults(self, REQUEST, **kw)
File "/Applications/Plone/buildout-cache/eggs/Products.ZCatalog-2.13.20-py2.6.egg/Products/ZCatalog/ZCatalog.py", line 604, in searchResults
return self._catalog.searchResults(REQUEST, used, **kw)
File "/Applications/Plone/buildout-cache/eggs/Products.ZCatalog-2.13.20-py2.6.egg/Products/ZCatalog/Catalog.py", line 909, in searchResults
return self.search(args, sort_index, reverse, sort_limit, _merge)
File "/Applications/Plone/buildout-cache/eggs/Products.ZCatalog-2.13.20-py2.6.egg/Products/ZCatalog/Catalog.py", line 658, in search
b_size=b_size)
File "/Applications/Plone/buildout-cache/eggs/Products.ZCatalog-2.13.20-py2.6.egg/Products/ZCatalog/Catalog.py", line 678, in sortResults
index_key_map = sort_index.documentToKeyMap()
File "/Applications/Plone/buildout-cache/eggs/plone.app.folder-1.0.4-py2.6.egg/plone/app/folder/nogopip.py", line 91, in documentToKeyMap
ids = folder.getOrdering().idsInOrder()
File "/Applications/Plone/buildout-cache/eggs/plone.folder-1.0.1-py2.6.egg/plone/folder/ordered.py", line 41, in getOrdering
adapter = getAdapter(self, IOrdering)
File "/Applications/Plone/buildout-cache/eggs/zope.component-3.9.5-py2.6.egg/zope/component/_api.py", line 96, in getAdapter
raise ComponentLookupError(object, interface, name)
zope.component.interfaces.ComponentLookupError: (<ATFolder at /Sanford Guide Web Edition/amt>, <InterfaceClass plone.folder.interfaces.IOrdering>, u'')
The best way is to do like the index itself.
Code snipped based on the CatalogTool (Products.CMFPlone)
from Acquisition import aq_inner
from Acquisition import aq_parent
from OFS.interfaces import IOrderedContainer
obj = brain.getObject()
parent = aq_parent(aq_inner(obj))
ordered = IOrderedContainer(parent, None)
if ordered is not None:
return ordered.getObjectPosition(obj.getId())
return 0