Alembic keeps giving me this error when I try to migrate my schema even though the initial migration went without a hitch.
sqlalchemy.exc.NoReferencedColumnError: Could not initialize target column for ForeignKey 'dataset.datasetid' on table 'analysis': table 'dataset' has no column named 'datasetid'
Here is a part of my models.py class
class Dataset(db.Model):
DatasetID = db.Column(db.Integer, primary_key = True)
SampleID = db.Column(db.String(50), db.ForeignKey('sample.SampleID', onupdate="cascade",ondelete="restrict"), nullable=False)
UploadDate = db.Column(db.Date, nullable=False)
UploadID = db.Column(db.Integer,db.ForeignKey('uploaders.UploadID', onupdate="cascade",ondelete="restrict"), nullable=False)
UploadStatus = db.Column(db.String(45), nullable=False)
HPFPath = db.Column(db.String(500))
DatasetType = db.Column(db.String(45), nullable=False)
SolvedStatus = db.Column(db.String(30), nullable=False)
InputFile = db.Column(db.Text)
RunID = db.Column(db.String(45))
Notes = db.Column(db.Text)
analyses = db.relationship('Analysis',backref='dataset',lazy='dynamic')
data2Cohorts = db.relationship('Dataset2Cohort',backref='dataset',lazy='dynamic')
class Dataset2Cohort(db.Model):
__tablename__='dataset2Cohort'
DatasetID = db.Column(db.Integer, db.ForeignKey('dataset.DatasetID', onupdate="cascade",ondelete="cascade"), nullable=False, primary_key = True)
CohortID = db.Column(db.Integer, db.ForeignKey('cohort.CohortID', onupdate="cascade", ondelete="restrict"), nullable=False, primary_key = True)
class Analysis(db.Model):
AnalysisID = db.Column(db.String(100), primary_key = True)
DatasetID = db.Column(db.Integer, db.ForeignKey('dataset.DatasetID', onupdate="cascade",ondelete="cascade"), nullable=False)
PipelineVersion = db.Column(db.String(30))
ResultsDirectory = db.Column(db.Text)
ResultsBAM = db.Column(db.Text)
AssignedTo = db.Column(db.String(100), nullable=True)
analysisStatuses = db.relationship('AnalysisStatus', backref='analysis', lazy='dynamic')
Does anyone know why I keep getting that error even though I have the DatasetID column in the Dataset table?
Thank you,
Teja.
Found a solution.
This seems to be an issue with how Mysql 8.x versions refer to column names in the foreign key declaration - Mysql 8.x versions always use lowercase when a column is referenced in Foreign Key statements, which cause an incompatibility with sqlalchemy. This issue is discussed here
https://github.com/sqlalchemy/sqlalchemy/issues/4344
Solution is to just upgrade the sqlalchemy to the latest version (>=1.2.x)
Teja.
Related
I'm trying to refactor a database model; separate one column out of a table into another new one. I'd like to do this using existing SQLAlchemy Core models & Alembic. I'd also like to use server-side INSERT ... FROM SELECT ...-style query to migrate data (docs). By avoiding having to copy all the gazillion of rows to Python-world I hope to have maximum scalability, maximum performance and minimum downtime.
My problem is the programmatic use of SQLAlchemy running on two versions of the same table name in a single Metadata context. Should I resort to using an textual SQL instead? 😕
schema.py before:
class User(Base):
__tablename__ = "users"
id = Column(BigInteger, primary_key=True, autoincrement=False, nullable=False)
[...]
profile_picture_url = Column(String, nullable=True)
schema.py after:
class User(Base):
__tablename__ = "users"
id = Column(BigInteger, primary_key=True, autoincrement=False, nullable=False)
[...]
class UserProfileExtras(Base):
__tablename__ = "user_profile_extras"
user_id = Column(BigInteger, ForeignKey("users.id"), index=True, nullable=False)
profile_picture_url = Column(String, nullable=False)
So here's my attempt to create an Alembic upgrade script:
# Import the new/current-in-code models.
from ... import User, UserProfileExtras
# Define the previous User model in order to operate on the current/old schema.
class UserBeforeUpgrade(Base):
__tablename__ = "users"
id = Column(BigInteger, primary_key=True, autoincrement=False, nullable=False)
[...]
profile_picture_url = Column(String, nullable=True)
table_before_upgrade: Table = UserBeforeUpgrade.__table__
new_target_table = UserProfileExtras.__table__
[...]
def upgrade() -> None:
op.create_table(
"user_profile_extras",
sa.Column("user_id", sa.BigInteger(), autoincrement=False, nullable=False),
sa.Column("profile_picture_url", sa.VARCHAR(), nullable=False),
[...]
)
from_user_table = (select([table_before_upgrade.c.id, table_before_upgrade.c.profile_picture_url])
.where(table_before_upgrade.c.profile_picture_url != None))
insert_from = (
new_target_table.insert().from_select(
[new_target_table.c.user_id, new_target_table.c.profile_picture_url],
from_user_table)
)
op.execute(insert_from))
[...]
[...]
Error:
sqlalchemy.exc.InvalidRequestError: Table 'users' is already defined for this MetaData instance.
Specify 'extend_existing=True' to redefine options and columns on an existing Table object.
I'm currently getting an error, I'm using sql server and trying to model a simple Parent with an array of Children:
sqlalchemy.exc.NoForeignKeysError: Could not determine join condition
between parent/child tables on relationship Parent.children- there are no
foreign keys linking these tables. Ensure that referencing columns
are associated with a ForeignKey or ForeignKeyConstraint, or specify a
'primaryjoin' expression.
my classes are set up simply as follows:
class Parent(db.Model):
__tablename__ = "parent"
parentId = db.Column(db.Integer, primary_key=True)
parentVersion = db.Column(db.Integer, primary_key=True)
children = db.relationship('Child', backref="parent",lazy=True)
class Child(db.Model):
__tablename__ = "child"
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(512), nullable=False)
parentId = db.Column(db.Integer, nullable=False)
parentVersion = db.Column(db.Integer, nullable=False)
ForeignKeyConstraint(['parentId', 'parentVersion'], ['parent.parentId', 'parent.parentVersion']
I've tried fiddling with declaring the relationship and foreign key in several ways but i always get an error, what is the correct way to do this?
Your forgot to add a foreign key:
parentId = db.Column(db.Integer, db.ForeignKey("parent.id))
There is a lot of documentation material regarding this topic, if there is still anything unclear to you.
You are missing adding the ForeignKeyConstraint to the table args, and you are using camel case, not snake case. And you don't need the __tablename__ with Flask-SQLAlchemy.
Try:
class Parent(db.Model):
id = db.Column(db.Integer, primary_key=True)
version = db.Column(db.Integer, primary_key=True)
children = db.relationship('Child', backref="parent", lazy=True)
class Child(db.Model):
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(512), nullable=False)
parent_id = db.Column(db.Integer, nullable=False)
parent_version = db.Column(db.Integer, nullable=False)
__table_args__ = (
db.ForeignKeyConstraint(
['parent_id', 'parent_version'],
['parent.id', 'parent.version']
),
)
Sorry if this sounds silly, but i'm trying to get all the books for an author. This is what I have:
class Author(db.Model):
__tablename__='author'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String)
class Book(db.Model):
__tablename__='book'
id = db.Column(db.Integer, primary_key=True)
title = db.Column(db.String)
author_id = db.Column(db.Integer, ForeignKey('author.id'))
author_rel = relationship('Author', lazy='joined'), backref='book')
and I have my schemas:
class BookSchema(Schema):
id = fields.Int()
name= field.Str()
class BookSchema(Schema):
id = fields.Int()
title = field.Str()
author = fields.Nested(Author)
So I can retrieve the books with the author and the authors.
What I need here is to add a nested field with all the books of each author... I've been trying, but failing to do so. Is there an automatic way to get this?
I'm trying to join the tables in the query, but also failing to do it:
session.query(Author, Book).join(Book, Author.id == Book.author_id).all()
This gives me a (Author, Book) tuple, and I cannot map that into a concise json... How could I do that?
EDIT:
Ok, so apparently I didn't understand the concept of a relationship haha I could avoid all this trouble by simply adding a relationship to my Author entity:
class Author(db.Model):
__tablename__='author'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String)
books = relationship('Book', lazy='joined', backref='author_book')
class Book(db.Model):
__tablename__='book'
id = db.Column(db.Integer, primary_key=True)
title = db.Column(db.String)
author_id = db.Column(db.Integer, ForeignKey('author.id'))
author = relationship('Author', lazy='joined', backref='book_author')
Then I could just populate my marshmallow schema normally and be happy
==================================================
The solution I used was:
session = Session()
author_objects = session.query(Author).all()
schema = AuthorSchema(many=True)
author_list = schema.dump(author_objects).data
book_objects = session.query(Book).all()
schema = BookSchema(many=True)
book_list = schema.dump(book_objects).data
for index, author in enumerate(author_list):
author_list[index]['books'] = [book for book in book_list if book['author_id'] == author['id']]
session.close()
return jsonify(author_list)
This feels kinda manual for me, I think there should be a better way to do this automatically using schemas. It's, after all, based on a relationship that exists.
This works, but would be slow for long lists... I preferred to do this using sql-alchemy + marshmallow directly...
Ideas?
This is newbee SQL question I struggled to find a clear answer to. So please help. I have two tables in database with fields as listed below:
ConstructionProjects
id (unique, nonempty)
developer (one, nonempty)
main_contractor (one, nonempty)
architect (one, nonempty)
(other fields)
Companies
id (unique, nonempty)
projects_developed (many or none)
projects_as_main_contractor (many or none)
projects_as_architect (many or none)
(other fields)
So every project has only one developer, one architect and one contractor, however, it may be the same company. Any company may be involved in as many projects in any roles.
Is there a way to avoid creating 3 additional association tables to establish many to many relationships? and make 3 one to many relationships instead?
If so, which practice is better?
*In other words, I don't understand relationships (one to many and many to many) relate (1) row to row or (2) row to "specific cell"?
(1) if row to row then I have many to many relationships
(2) if row to specific cell then it is multiple one to many relationships...*
I'm learning Flask_alchemy and PostgreSQL.
I ran into problem, writing a code like this (there's no reference to specific columns between tables). So this is not ok?
class Company(db.Model):
id = db.Column(db.Integer, primary_key = True)
constr_projects_developed = db.relationship('ConstrProject', backref='developer')
constr_projects_main_contracts = db.relationship('ConstrProject', backref='main_contractor')
constr_projects_architect = db.relationship('ConstrProject', backref='architect')
class ConstrProject(db.Model):
id = db.Column(db.Integer, primary_key = True)
developer_id = db.Column(db.Integer, db.ForeignKey('company.id'))
main_contractor_id = db.Column(db.Integer, db.ForeignKey('company.id'))
architect_id = db.Column(db.Integer, db.ForeignKey('company.id'))
Then my question is, the correct way to do it is like this (1):
class Company(db.Model):
id = db.Column(db.Integer, primary_key = True)
constr_projects_developed = db.relationship('ConstrProject', back_populates='developer')
constr_projects_main_contracts = db.relationship('ConstrProject', back_populates='main_contractor')
constr_projects_architect = db.relationship('ConstrProject', back_populates='architect')
class ConstrProject(db.Model):
id = db.Column(db.Integer, primary_key = True)
developer_id = db.Column(db.Integer, db.ForeignKey('company.id'))
developer = db.relationship('Company', back_populates='constr_projects_developed')
main_contractor_id = db.Column(db.Integer, db.ForeignKey('company.id'))
main contractor = db.relationship('Company', back_populates='constr_projects_main_contracts')
architect_id = db.Column(db.Integer, db.ForeignKey('company.id'))
architect = db.relationship('Company', back_populates='constr_projects_architect')
Or like this(2)?:
class Company(db.Model):
id = db.Column(db.Integer, primary_key = True)
cp_developed = db.relationship('Company', secondary=cp_developer_company, back_populates='developer')
cp_main_contracts = db.relationship('Company', secondary=cp_main_contractor_company, back_populates='main_contractor')
cp_architects = db.relationship('Company', secondary=cp_architect_company, back_populates='architect')
class ConstrProject(db.Model):
id = db.Column(db.Integer, primary_key = True)
developer = db.relationship('Company', secondary=cp_developer_company, back_populates='cp_developed')
main_contractor = db.relationship('Company', secondary=cp_main_contractor_company, back_populates='cp_main_contracts')
architect = db.relationship('Company', secondary=cp_architect_company, back_populates='cp_architects')
cp_developer_company = db.Table('cp_developer_company'
db.Column('company_id', db.Integer, db.ForeignKey('company.id'))
db.Column('constr_project_id', db.Integer, db.ForeignKey('constrproject.id'))
)
cp_main_contractor_company = db.Table('cp_main_contractor_company'
db.Column('company_id', db.Integer, db.ForeignKey('company.id'))
db.Column('constr_project_id', db.Integer, db.ForeignKey('constrproject.id'))
)
cp_architect_company = db.Table('cp_architect_company'
db.Column('company_id', db.Integer, db.ForeignKey('company.id'))
db.Column('constr_project_id', db.Integer, db.ForeignKey('constrproject.id'))
With the following scheme:
class User(Base):
id = Column(Integer, primary_key=True)
name = Column(String)
class Photo(Base):
id = Column(Integer, primary_key=True)
user_id = Column(Integer, ForeignKey(User.id), nullable=False)
user = relationship(User)
class Tag(Base):
id = Column(Integer, primary_key=True)
tag_name = Column(String)
tag_version = Column(Integer)
photo_id = Column(Integer, ForeignKey(Photo.id), nullable=False)
photo = relationship(Photo)
How do I create an SQLAlchemy query to get all the photos of a specific user, that don't have a specific tag and version.
As in "all the photos of the user with id "1234" that don't have a "cat" of version "2" tagged in them".
Also interesting would be "all the users who have at least one photo without a specific tag"
I'm using postgreSQL btw.
Here is a complete example that sets up relationships, creates some sample data, then performs your two queries.
Setup:
from datetime import datetime
from sqlalchemy import create_engine, Column, Integer, String, ForeignKey, not_
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker, relationship
engine = create_engine('sqlite:///', echo=True)
Session = sessionmaker(bind=engine)
session = Session()
Base = declarative_base(bind=engine)
class User(Base):
__tablename__ = 'user'
id = Column(Integer, primary_key=True)
name = Column(String, nullable=False)
class Photo(Base):
__tablename__ = 'photo'
id = Column(Integer, primary_key=True)
name = Column(String, nullable=False)
user_id = Column(Integer, ForeignKey(User.id), nullable=False)
user = relationship(User, backref='photos')
class Tag(Base):
__tablename__ = 'tag'
id = Column(Integer, primary_key=True)
name = Column(String, nullable=False)
photo_id = Column(Integer, ForeignKey(Photo.id), nullable=False)
photo = relationship(Photo, backref='tags')
Base.metadata.create_all()
session.add(User(name='davidism', photos=[
Photo(name='sun', tags=[Tag(name='bright'), Tag(name='day')]),
Photo(name='moon', tags=[Tag(name='bright'), Tag(name='night')])
]))
session.add(User(name='eran', photos=[
Photo(name='party', tags=[Tag(name='people'), Tag(name='night')]),
Photo(name='cat')
]))
session.commit()
Query all photos with no tags at all:
no_tags = session.query(Photo).outerjoin(Photo.tags).filter(not_(Photo.tags.any())).all()
print 'no tags: ', len(no_tags)
Query all photos without the tag 'night':
not_night = session.query(Photo).outerjoin(Photo.tags).filter(not_(Photo.tags.any(Tag.name == 'night'))).all()
print 'not night: ', len(not_night)
Assuming existance of backrefs Tag.photo = relationship(Photo, backref='tags') and
Photo.user = relationship(User, backref="photos") both can be done using any construct. This might not generate the most optimal SQL SELECT statement, but it is a very clean sqlalchemy.
Part-1: "all the photos of the user with id "1234" that don't have a "cat" of version "2" tagged in them"
def get_user_photos_without_tag(user_id, tag_name, tag_version):
qry = (session.query(Photo)
.filter(~Photo.tags.any(and_(
Tag.tag_name == tag_name,
Tag.tag_version == tag_version))
)
.filter(Photo.user_id == user_id)
)
return qry.all()
photos = get_user_photos_without_tag(1234, 'cat', 2)
Part-2: "all the users who have at least one photo without a specific tag"
def get_user_with_photos_without_tag(tag_name, tag_version):
qry = (session.query(User)
.filter(User.photos.any(
~Photo.tags.any(and_(
Tag.tag_name == tag_name,
Tag.tag_version == tag_version))
))
)
return qry.all()
res = get_user_with_photos_without_tag('cat', 2)