Skip to content

perf: use bulk insert to ahb_expressions; remove debug logs #216

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hf-kklein merged 2 commits into from
Dec 4, 2025
Merged

perf: use bulk insert to ahb_expressions; remove debug logs #216

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a0156d7
perf: use bulk insert to `ahb_expressions`; remove debug logs
Dec 4, 2025
e89a6de
pylint
Dec 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 13 additions & 20 deletions src/fundamend/sqlmodels/expression_view.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
from fundamend.sqlmodels.anwendungshandbuch import Paket, UbBedingung

try:
from sqlalchemy.sql.functions import func
from sqlmodel import Field, Session, SQLModel, UniqueConstraint, col, select

except ImportError as import_error:
Expand Down Expand Up @@ -186,6 +185,7 @@ def create_and_fill_ahb_expression_table(session: Session, use_cpu_intensive_val
if (key := (row[0], row[1], row[2].strip())) not in seen
and not seen.add(key) # type:ignore[ func-returns-value]
]
ahb_expression_rows: list[AhbExpression] = []
for row in unique_rows: # there are ~3600 unique rows for FV2410+FV2504 as of 2025-04-15
expression = row[2].strip()
if use_cpu_intensive_validity_check:
Expand All @@ -195,30 +195,23 @@ def create_and_fill_ahb_expression_table(session: Session, use_cpu_intensive_val
)
else:
_, node_texts, error_message = _get_validity_node_texts_and_error_message_fast(expression, session, row[3])
ahb_expression_row = AhbExpression(
edifact_format_version=row[0],
format=row[1],
expression=expression,
node_texts=node_texts,
anwendungshandbuch_primary_key=row[3],
ahbicht_error_message=error_message,
)
session.add(ahb_expression_row)
_logger.debug(
"Added row (%s, %s, %s) to the ahb_expressions_table",
ahb_expression_row.edifact_format_version,
ahb_expression_row.format,
ahb_expression_row.expression,
ahb_expression_rows.append(
AhbExpression(
edifact_format_version=row[0],
format=row[1],
expression=expression,
node_texts=node_texts,
anwendungshandbuch_primary_key=row[3],
ahbicht_error_message=error_message,
)
)
number_of_inserted_rows = session.scalar(
select(func.count(AhbExpression.id)) # type:ignore[arg-type]# pylint:disable=not-callable
)
session.add_all(ahb_expression_rows)
session.commit()
_logger.info(
"Inserted %d rows into the table %s",
number_of_inserted_rows,
len(ahb_expression_rows),
AhbExpression.__tablename__,
)
session.commit()


class AhbExpression(SQLModel, table=True):
Expand Down