You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

db_manager.py 17 kB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406
  1. import threading
  2. from datetime import datetime
  3. from pathlib import Path
  4. from typing import Optional
  5. from loguru import logger
  6. from sqlalchemy import exc, func, inspect, text
  7. from sqlmodel import Session, SQLModel, and_, create_engine, select
  8. from ..datamodel import LinkTypes, Response
  9. from .schema_manager import SchemaManager
  10. # from .dbutils import init_db_samples
  11. class DatabaseManager:
  12. _init_lock = threading.Lock()
  13. def __init__(self, engine_uri: str, base_dir: Optional[Path] = None):
  14. """
  15. Initialize DatabaseManager with database connection settings.
  16. Does not perform any database operations.
  17. Args:
  18. engine_uri: Database connection URI (e.g. sqlite:///db.sqlite3)
  19. base_dir: Base directory for migration files. If None, uses current directory
  20. """
  21. connection_args = {"check_same_thread": True} if "sqlite" in engine_uri else {}
  22. self.engine = create_engine(engine_uri, connect_args=connection_args)
  23. self.schema_manager = SchemaManager(
  24. engine=self.engine,
  25. base_dir=base_dir,
  26. )
  27. def initialize_database(self, auto_upgrade: bool = False, force_init_alembic: bool = True) -> Response:
  28. """
  29. Initialize database and migrations in the correct order.
  30. Args:
  31. auto_upgrade: If True, automatically generate and apply migrations for schema changes
  32. force_init_alembic: If True, reinitialize alembic configuration even if it exists
  33. """
  34. if not self._init_lock.acquire(blocking=False):
  35. return Response(message="Database initialization already in progress", status=False)
  36. try:
  37. inspector = inspect(self.engine)
  38. tables_exist = inspector.get_table_names()
  39. if not tables_exist:
  40. # Fresh install - create tables and initialize migrations
  41. logger.info("Creating database tables...")
  42. SQLModel.metadata.create_all(self.engine)
  43. if self.schema_manager.initialize_migrations(force=force_init_alembic):
  44. return Response(message="Database initialized successfully", status=True)
  45. return Response(message="Failed to initialize migrations", status=False)
  46. # Handle existing database
  47. if auto_upgrade:
  48. logger.info("Checking database schema...")
  49. if self.schema_manager.ensure_schema_up_to_date(): # <-- Use this instead
  50. return Response(message="Database schema is up to date", status=True)
  51. return Response(message="Database upgrade failed", status=False)
  52. return Response(message="Database is ready", status=True)
  53. except Exception as e:
  54. error_msg = f"Database initialization failed: {str(e)}"
  55. logger.error(error_msg)
  56. return Response(message=error_msg, status=False)
  57. finally:
  58. self._init_lock.release()
  59. def reset_db(self, recreate_tables: bool = True):
  60. """
  61. Reset the database by dropping all tables and optionally recreating them.
  62. Args:
  63. recreate_tables (bool): If True, recreates the tables after dropping them.
  64. Set to False if you want to call create_db_and_tables() separately.
  65. """
  66. if not self._init_lock.acquire(blocking=False):
  67. logger.warning("Database reset already in progress")
  68. return Response(message="Database reset already in progress", status=False, data=None)
  69. try:
  70. # Dispose existing connections
  71. self.engine.dispose()
  72. with Session(self.engine) as session:
  73. try:
  74. # Disable foreign key checks for SQLite
  75. if "sqlite" in str(self.engine.url):
  76. session.exec(text("PRAGMA foreign_keys=OFF"))
  77. # Drop all tables
  78. SQLModel.metadata.drop_all(self.engine)
  79. logger.info("All tables dropped successfully")
  80. # Re-enable foreign key checks for SQLite
  81. if "sqlite" in str(self.engine.url):
  82. session.exec(text("PRAGMA foreign_keys=ON"))
  83. session.commit()
  84. except Exception as e:
  85. session.rollback()
  86. raise e
  87. finally:
  88. session.close()
  89. self._init_lock.release()
  90. if recreate_tables:
  91. logger.info("Recreating tables...")
  92. self.initialize_database(auto_upgrade=False, force_init_alembic=True)
  93. return Response(
  94. message="Database reset successfully" if recreate_tables else "Database tables dropped successfully",
  95. status=True,
  96. data=None,
  97. )
  98. except Exception as e:
  99. error_msg = f"Error while resetting database: {str(e)}"
  100. logger.error(error_msg)
  101. return Response(message=error_msg, status=False, data=None)
  102. finally:
  103. if self._init_lock.locked():
  104. self._init_lock.release()
  105. logger.info("Database reset lock released")
  106. def upsert(self, model: SQLModel, return_json: bool = True):
  107. """Create or update an entity
  108. Args:
  109. model (SQLModel): The model instance to create or update
  110. return_json (bool, optional): If True, returns the model as a dictionary.
  111. If False, returns the SQLModel instance. Defaults to True.
  112. Returns:
  113. Response: Contains status, message and data (either dict or SQLModel based on return_json)
  114. """
  115. status = True
  116. model_class = type(model)
  117. existing_model = None
  118. with Session(self.engine) as session:
  119. try:
  120. existing_model = session.exec(select(model_class).where(model_class.id == model.id)).first()
  121. if existing_model:
  122. model.updated_at = datetime.now()
  123. for key, value in model.model_dump().items():
  124. setattr(existing_model, key, value)
  125. model = existing_model # Use the updated existing model
  126. session.add(model)
  127. else:
  128. session.add(model)
  129. session.commit()
  130. session.refresh(model)
  131. except Exception as e:
  132. session.rollback()
  133. logger.error("Error while updating/creating " + str(model_class.__name__) + ": " + str(e))
  134. status = False
  135. return Response(
  136. message=(
  137. f"{model_class.__name__} Updated Successfully"
  138. if existing_model
  139. else f"{model_class.__name__} Created Successfully"
  140. ),
  141. status=status,
  142. data=model.model_dump() if return_json else model,
  143. )
  144. def _model_to_dict(self, model_obj):
  145. return {col.name: getattr(model_obj, col.name) for col in model_obj.__table__.columns}
  146. def get(
  147. self,
  148. model_class: SQLModel,
  149. filters: dict = None,
  150. return_json: bool = False,
  151. order: str = "desc",
  152. ):
  153. """List entities"""
  154. with Session(self.engine) as session:
  155. result = []
  156. status = True
  157. status_message = ""
  158. try:
  159. statement = select(model_class)
  160. if filters:
  161. conditions = [getattr(model_class, col) == value for col, value in filters.items()]
  162. statement = statement.where(and_(*conditions))
  163. if hasattr(model_class, "created_at") and order:
  164. order_by_clause = getattr(model_class.created_at, order)() # Dynamically apply asc/desc
  165. statement = statement.order_by(order_by_clause)
  166. items = session.exec(statement).all()
  167. result = [self._model_to_dict(item) if return_json else item for item in items]
  168. status_message = f"{model_class.__name__} Retrieved Successfully"
  169. except Exception as e:
  170. session.rollback()
  171. status = False
  172. status_message = f"Error while fetching {model_class.__name__}"
  173. logger.error("Error while getting items: " + str(model_class.__name__) + " " + str(e))
  174. return Response(message=status_message, status=status, data=result)
  175. def delete(self, model_class: SQLModel, filters: dict = None):
  176. """Delete an entity"""
  177. status_message = ""
  178. status = True
  179. with Session(self.engine) as session:
  180. try:
  181. statement = select(model_class)
  182. if filters:
  183. conditions = [getattr(model_class, col) == value for col, value in filters.items()]
  184. statement = statement.where(and_(*conditions))
  185. rows = session.exec(statement).all()
  186. if rows:
  187. for row in rows:
  188. session.delete(row)
  189. session.commit()
  190. status_message = f"{model_class.__name__} Deleted Successfully"
  191. else:
  192. status_message = "Row not found"
  193. logger.info(f"Row with filters {filters} not found")
  194. except exc.IntegrityError as e:
  195. session.rollback()
  196. status = False
  197. status_message = f"Integrity error: The {model_class.__name__} is linked to another entity and cannot be deleted. {e}"
  198. # Log the specific integrity error
  199. logger.error(status_message)
  200. except Exception as e:
  201. session.rollback()
  202. status = False
  203. status_message = f"Error while deleting: {e}"
  204. logger.error(status_message)
  205. return Response(message=status_message, status=status, data=None)
  206. def link(
  207. self,
  208. link_type: LinkTypes,
  209. primary_id: int,
  210. secondary_id: int,
  211. sequence: Optional[int] = None,
  212. ):
  213. """Link two entities with automatic sequence handling."""
  214. with Session(self.engine) as session:
  215. try:
  216. # Get classes from LinkTypes
  217. primary_class = link_type.primary_class
  218. secondary_class = link_type.secondary_class
  219. link_table = link_type.link_table
  220. # Get entities
  221. primary_entity = session.get(primary_class, primary_id)
  222. secondary_entity = session.get(secondary_class, secondary_id)
  223. if not primary_entity or not secondary_entity:
  224. return Response(message="One or both entities do not exist", status=False)
  225. # Get field names
  226. primary_id_field = f"{primary_class.__name__.lower()}_id"
  227. secondary_id_field = f"{secondary_class.__name__.lower()}_id"
  228. # Check for existing link
  229. existing_link = session.exec(
  230. select(link_table).where(
  231. and_(
  232. getattr(link_table, primary_id_field) == primary_id,
  233. getattr(link_table, secondary_id_field) == secondary_id,
  234. )
  235. )
  236. ).first()
  237. if existing_link:
  238. return Response(message="Link already exists", status=False)
  239. # Get the next sequence number if not provided
  240. if sequence is None:
  241. max_seq_result = session.exec(
  242. select(func.max(link_table.sequence)).where(getattr(link_table, primary_id_field) == primary_id)
  243. ).first()
  244. sequence = 0 if max_seq_result is None else max_seq_result + 1
  245. # Create new link
  246. new_link = link_table(
  247. **{primary_id_field: primary_id, secondary_id_field: secondary_id, "sequence": sequence}
  248. )
  249. session.add(new_link)
  250. session.commit()
  251. return Response(message=f"Entities linked successfully with sequence {sequence}", status=True)
  252. except Exception as e:
  253. session.rollback()
  254. return Response(message=f"Error linking entities: {str(e)}", status=False)
  255. def unlink(self, link_type: LinkTypes, primary_id: int, secondary_id: int, sequence: Optional[int] = None):
  256. """Unlink two entities and reorder sequences if needed."""
  257. with Session(self.engine) as session:
  258. try:
  259. # Get classes from LinkTypes
  260. primary_class = link_type.primary_class
  261. secondary_class = link_type.secondary_class
  262. link_table = link_type.link_table
  263. # Get field names
  264. primary_id_field = f"{primary_class.__name__.lower()}_id"
  265. secondary_id_field = f"{secondary_class.__name__.lower()}_id"
  266. # Find existing link
  267. statement = select(link_table).where(
  268. and_(
  269. getattr(link_table, primary_id_field) == primary_id,
  270. getattr(link_table, secondary_id_field) == secondary_id,
  271. )
  272. )
  273. if sequence is not None:
  274. statement = statement.where(link_table.sequence == sequence)
  275. existing_link = session.exec(statement).first()
  276. if not existing_link:
  277. return Response(message="Link does not exist", status=False)
  278. deleted_sequence = existing_link.sequence
  279. session.delete(existing_link)
  280. # Reorder sequences for remaining links
  281. remaining_links = session.exec(
  282. select(link_table)
  283. .where(getattr(link_table, primary_id_field) == primary_id)
  284. .where(link_table.sequence > deleted_sequence)
  285. .order_by(link_table.sequence)
  286. ).all()
  287. # Decrease sequence numbers to fill the gap
  288. for link in remaining_links:
  289. link.sequence -= 1
  290. session.commit()
  291. return Response(message="Entities unlinked successfully and sequences reordered", status=True)
  292. except Exception as e:
  293. session.rollback()
  294. return Response(message=f"Error unlinking entities: {str(e)}", status=False)
  295. def get_linked_entities(
  296. self,
  297. link_type: LinkTypes,
  298. primary_id: int,
  299. return_json: bool = False,
  300. ):
  301. """Get linked entities based on link type and primary ID, ordered by sequence."""
  302. with Session(self.engine) as session:
  303. try:
  304. # Get classes from LinkTypes
  305. primary_class = link_type.primary_class
  306. secondary_class = link_type.secondary_class
  307. link_table = link_type.link_table
  308. # Get field names
  309. primary_id_field = f"{primary_class.__name__.lower()}_id"
  310. secondary_id_field = f"{secondary_class.__name__.lower()}_id"
  311. # Query both link and entity, ordered by sequence
  312. items = session.exec(
  313. select(secondary_class)
  314. .join(link_table, getattr(link_table, secondary_id_field) == secondary_class.id)
  315. .where(getattr(link_table, primary_id_field) == primary_id)
  316. .order_by(link_table.sequence)
  317. ).all()
  318. result = [item.model_dump() if return_json else item for item in items]
  319. return Response(message="Linked entities retrieved successfully", status=True, data=result)
  320. except Exception as e:
  321. logger.error(f"Error getting linked entities: {str(e)}")
  322. return Response(message=f"Error getting linked entities: {str(e)}", status=False, data=[])
  323. # Add new close method
  324. async def close(self):
  325. """Close database connections and cleanup resources"""
  326. logger.info("Closing database connections...")
  327. try:
  328. # Dispose of the SQLAlchemy engine
  329. self.engine.dispose()
  330. logger.info("Database connections closed successfully")
  331. except Exception as e:
  332. logger.error(f"Error closing database connections: {str(e)}")
  333. raise