From 5e4c28254f8faa338930004ffa284fecddf977b5 Mon Sep 17 00:00:00 2001 From: Marty Pradere Date: Tue, 18 Aug 2026 17:42:19 -0600 Subject: [PATCH 1/6] Log EventDataIds through the SND event and attribute data ETL steps The _SND Event Data step clears attribute values and only the _SND Attribute Data step restores them, and each step picks its rows independently, so a mismatch silently leaves event data with no attributes. Logging the ids on both sides makes the gap visible in the job log. --- snd/src/org/labkey/snd/SNDManager.java | 27 +++++++++ .../labkey/snd/query/AttributeDataTable.java | 59 +++++++++++++++++-- .../org/labkey/snd/query/EventDataTable.java | 55 ++++++++++++++++- 3 files changed, 133 insertions(+), 8 deletions(-) diff --git a/snd/src/org/labkey/snd/SNDManager.java b/snd/src/org/labkey/snd/SNDManager.java index b6f37af5b..a7074ff96 100644 --- a/snd/src/org/labkey/snd/SNDManager.java +++ b/snd/src/org/labkey/snd/SNDManager.java @@ -105,6 +105,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.TreeMap; @@ -153,6 +154,9 @@ public static UserSchema getSndUserSchemaAdminRole(Container c, User u) public static int MAX_MERGE_ROWS = 2000; + private static final int MAX_LOGGED_IDS = 5000; + private static final int LOGGED_IDS_PER_LINE = 250; + public static Logger getLogger(Map configParameters, Class clazz) { Logger log = null; @@ -164,6 +168,29 @@ public static Logger getLogger(Map configParameters, Class claz return log; } + /** + * Writes an id set to the ETL job log so that the id sets logged by different ETL steps of the same run can be + * diffed against each other. Chunked because a single line of thousands of ids is unreadable, and capped because + * the initial full data load would otherwise write the entire table to the log. + */ + public static void logIds(Logger log, String message, Collection ids) + { + log.info(message + " Count: " + ids.size() + "."); + + if (ids.isEmpty()) + return; + + if (ids.size() > MAX_LOGGED_IDS) + { + log.info("Id list omitted, more than " + MAX_LOGGED_IDS + " ids."); + return; + } + + List sorted = ids.stream().filter(Objects::nonNull).sorted().collect(Collectors.toList()); + for (List chunk : ListUtils.partition(sorted, LOGGED_IDS_PER_LINE)) + log.info(" " + StringUtils.join(chunk, ", ")); + } + public static String getPackageName(int id) { return PackageDomainKind.getPackageKindName() + "-" + id; diff --git a/snd/src/org/labkey/snd/query/AttributeDataTable.java b/snd/src/org/labkey/snd/query/AttributeDataTable.java index 81d343671..18f4ba029 100644 --- a/snd/src/org/labkey/snd/query/AttributeDataTable.java +++ b/snd/src/org/labkey/snd/query/AttributeDataTable.java @@ -149,6 +149,10 @@ public QueryUpdateService getUpdateService() protected class UpdateService extends SNDQueryUpdateService { + /** Bounds the source ordering check below. It costs one retained URI per distinct EventDataId, and an ungrouped source would otherwise warn once per row. */ + private static final int MAX_TRACKED_URIS = 50_000; + private static final int MAX_ORDER_WARNINGS = 10; + private final SNDManager _sndManager = SNDManager.get(); private final SNDService _sndService = SNDService.get(); private final DbSchema _expSchema = OntologyManager.getExpSchema(); @@ -230,7 +234,11 @@ private int insertObject(Container c, User u, String uri, List p private List> updateObjectProperty(User user, Container container, List> data, boolean isInsertOnly, boolean isUpdate, Logger logger) { - logger.info("Begin updating exp.ObjectProperty."); + Set incomingEventDataIds = new HashSet<>(); + for (Map row : data) + incomingEventDataIds.add((Integer) row.get("EventDataId")); + + SNDManager.logIds(logger, "Begin updating exp.ObjectProperty. Source rows: " + data.size() + ". EventDataIds in this batch:", incomingEventDataIds); int inserted = 0; @@ -240,6 +248,10 @@ private List> updateObjectProperty(User user, Container cont boolean found = false; Set cacheEventIds = new HashSet<>(); + Set writtenEventDataIds = new HashSet<>(); + Set flushedUris = new HashSet<>(); + boolean checkOrdering = true; + int outOfOrderFlushes = 0; for(Map row : data) { @@ -255,7 +267,8 @@ private List> updateObjectProperty(User user, Container cont //add to list of cached narrative rows to delete cacheEventIds.add((Integer) row.get("EventId")); - String objectURI = getObjectURI((Integer) row.get("EventDataId"), container); + Integer eventDataId = (Integer) row.get("EventDataId"); + String objectURI = getObjectURI(eventDataId, container); if (prevUri == null) prevUri = objectURI; @@ -320,10 +333,27 @@ else if (stringValue != null) if (!prevUri.equals(objectURI)) { inserted = insertObject(container, user, prevUri, prevObjProps, pkgId, inserted, logger); + + // Properties are only flushed when the URI changes, so a URI seen twice means the source + // did not arrive grouped by EventDataId and the ORDER BY in v_snd_attributeData was lost. + if (checkOrdering) + { + if (!flushedUris.add(prevUri) && ++outOfOrderFlushes <= MAX_ORDER_WARNINGS) + logger.warn("Source rows are not grouped by EventDataId; exp.ObjectProperty for {} was written in more than one pass.", prevUri); + + if (flushedUris.size() >= MAX_TRACKED_URIS) + { + logger.info("More than {} EventDataIds in this batch; ending the source ordering check.", MAX_TRACKED_URIS); + flushedUris.clear(); + checkOrdering = false; + } + } + prevUri = objectURI; prevObjProps = new ArrayList<>(); } prevObjProps.add(oprop); + writtenEventDataIds.add(eventDataId); } } @@ -332,12 +362,16 @@ else if (stringValue != null) } if (!found) { - throw new RuntimeException("Attribute metadata not found for key: '" + key + "' in package: " + pkgId); + throw new RuntimeException("Attribute metadata not found for key: '" + key + "' in package: " + pkgId + + ", EventDataId: " + eventDataId + ". Aborting, leaving all " + incomingEventDataIds.size() + + " EventDataIds in this batch with the attribute values the _SND Event Data step already cleared."); } } else { - throw new RuntimeException("Package metadata not found for package id: " + pkgId); + throw new RuntimeException("Package metadata not found for package id: " + pkgId + + ", EventDataId: " + eventDataId + ". Aborting, leaving all " + incomingEventDataIds.size() + + " EventDataIds in this batch with the attribute values the _SND Event Data step already cleared."); } } @@ -347,7 +381,22 @@ else if (stringValue != null) } OntologyManager.clearPropertyCache(); - logger.info("End updating exp.ObjectProperty. Inserted/Updated " + inserted + " rows."); + + SNDManager.logIds(logger, "End updating exp.ObjectProperty. Inserted/Updated " + inserted + " rows. EventDataIds written:", writtenEventDataIds); + + if (outOfOrderFlushes > MAX_ORDER_WARNINGS) + logger.warn("{} objectURIs in total were written in more than one pass; further warnings were suppressed.", outOfOrderFlushes); + + // Collect only the misses; copying the incoming set would double its footprint on a full load. + Set unwritten = new HashSet<>(); + for (Integer id : incomingEventDataIds) + { + if (!writtenEventDataIds.contains(id)) + unwritten.add(id); + } + + if (!unwritten.isEmpty()) + SNDManager.logIds(logger, "EventDataIds present in the source rows but left with no attribute values written:", unwritten); _sndManager.updateNarrativeCache(container, user, cacheEventIds, logger); diff --git a/snd/src/org/labkey/snd/query/EventDataTable.java b/snd/src/org/labkey/snd/query/EventDataTable.java index 1c9bce9d6..d6ca1ad6f 100644 --- a/snd/src/org/labkey/snd/query/EventDataTable.java +++ b/snd/src/org/labkey/snd/query/EventDataTable.java @@ -25,6 +25,7 @@ import org.labkey.api.data.JdbcType; import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.SqlSelector; import org.labkey.api.data.TableInfo; import org.labkey.api.dataiterator.DataIteratorBuilder; import org.labkey.api.dataiterator.DataIteratorContext; @@ -48,7 +49,9 @@ import java.io.IOException; import java.sql.SQLException; +import java.util.ArrayList; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -108,6 +111,9 @@ public QueryUpdateService getUpdateService() protected static class UpdateService extends SNDQueryUpdateService { + /** Keeps the ObjectURI IN clause well under the SQL Server parameter limit. */ + private static final int URI_CHUNK_SIZE = 500; + private final SNDManager _sndManager = SNDManager.get(); private final SNDService _sndService = SNDService.get(); private final DbSchema _expSchema = OntologyManager.getExpSchema(); @@ -122,6 +128,36 @@ private String getObjectURI(Integer eventDataId, Container c) return _sndManager.generateLsid(c, String.valueOf(eventDataId)); } + /** + * EventDataIds in this batch whose exp.Object currently carries attribute values. Deleting the exp.Object row + * cascades to exp.ObjectProperty, so these are the values the merge destroys; only the _SND Attribute Data ETL + * step re-inserts them, and it computes its incremental window independently of this step's. + */ + private Set getEventDataIdsWithAttributeData(Container container, Map eventDataIdsByUri) + { + Set withAttributeData = new HashSet<>(); + List uris = new ArrayList<>(eventDataIdsByUri.keySet()); + + for (int i = 0; i < uris.size(); i += URI_CHUNK_SIZE) + { + List chunk = uris.subList(i, Math.min(i + URI_CHUNK_SIZE, uris.size())); + + // EXISTS rather than a join: both indexes (UQ_Object on ObjectURI, PK_ObjectProperty on ObjectId) + // are seeks, and the semi-join stops at the first property instead of reading all of them per object. + SQLFragment sql = new SQLFragment("SELECT o.ObjectURI FROM ") + .append(OntologyManager.getTinfoObject(), "o") + .append(" WHERE o.Container = ?").add(container.getId()) + .append(" AND EXISTS (SELECT 1 FROM ").append(OntologyManager.getTinfoObjectProperty(), "op") + .append(" WHERE op.ObjectId = o.ObjectId)") + .append(" AND o.ObjectURI").appendInClause(chunk, _expSchema.getSqlDialect()); + + new SqlSelector(_expSchema, sql).getCollection(String.class) + .forEach(uri -> withAttributeData.add(eventDataIdsByUri.get(uri))); + } + + return withAttributeData; + } + @Override public int mergeRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) @@ -158,14 +194,27 @@ public int mergeRows(User user, Container container, DataIteratorBuilder rows, B log.info("Merging rows."); log.info("Begin updating exp.Object table."); - int count = 0; - for(Map map : data) + + Map eventDataIdsByUri = new LinkedHashMap<>(); + for (Map map : data) { - String objectURI = getObjectURI((Integer) map.get("EventDataId"), container); + Integer eventDataId = (Integer) map.get("EventDataId"); + String objectURI = getObjectURI(eventDataId, container); //update snd.EventData row with objectURI map.put("ObjectURI", objectURI); + eventDataIdsByUri.put(objectURI, eventDataId); + } + + SNDManager.logIds(log, "Attribute values about to be cleared by this merge; the _SND Attribute Data step must re-insert them.", + getEventDataIdsWithAttributeData(container, eventDataIdsByUri)); + + int count = 0; + for(Map map : data) + { + String objectURI = (String) map.get("ObjectURI"); + //delete row from exp.Object OntologyManager.deleteOntologyObjects(container, objectURI); From 23c45e35e3f4757ac80fa4de4929df0ccbf887a8 Mon Sep 17 00:00:00 2001 From: Marty Pradere Date: Tue, 18 Aug 2026 19:24:10 -0600 Subject: [PATCH 2/6] Keep the SND attribute-clearing diagnostic off the merge critical path The exp.Object lookup added for this branch's id logging ran unguarded inside mergeRows, so a failure reading it would abort a merge that previously did no reads there at all. It now runs inside logAttributeDataToBeCleared, which logs a warning and lets the merge continue. The lookup is also skipped once the batch exceeds MAX_LOGGED_IDS, the same threshold above which logIds suppresses the id list, capping the chunked queries at ten round trips instead of the hundreds a full merge batch would spend to produce a bare count. MAX_LOGGED_IDS is public now so both sites share one constant. --- snd/src/org/labkey/snd/SNDManager.java | 2 +- .../org/labkey/snd/query/EventDataTable.java | 26 +++++++++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/snd/src/org/labkey/snd/SNDManager.java b/snd/src/org/labkey/snd/SNDManager.java index a7074ff96..9a2e43805 100644 --- a/snd/src/org/labkey/snd/SNDManager.java +++ b/snd/src/org/labkey/snd/SNDManager.java @@ -154,7 +154,7 @@ public static UserSchema getSndUserSchemaAdminRole(Container c, User u) public static int MAX_MERGE_ROWS = 2000; - private static final int MAX_LOGGED_IDS = 5000; + public static final int MAX_LOGGED_IDS = 5000; private static final int LOGGED_IDS_PER_LINE = 250; public static Logger getLogger(Map configParameters, Class clazz) diff --git a/snd/src/org/labkey/snd/query/EventDataTable.java b/snd/src/org/labkey/snd/query/EventDataTable.java index d6ca1ad6f..dfe340a63 100644 --- a/snd/src/org/labkey/snd/query/EventDataTable.java +++ b/snd/src/org/labkey/snd/query/EventDataTable.java @@ -158,6 +158,29 @@ private Set getEventDataIdsWithAttributeData(Container container, Map eventDataIdsByUri, Logger log) + { + if (eventDataIdsByUri.size() > SNDManager.MAX_LOGGED_IDS) + { + log.info("More than " + SNDManager.MAX_LOGGED_IDS + " EventDataIds in this batch; skipping the check for attribute values about to be cleared."); + return; + } + + try + { + SNDManager.logIds(log, "Attribute values about to be cleared by this merge; the _SND Attribute Data step must re-insert them.", + getEventDataIdsWithAttributeData(container, eventDataIdsByUri)); + } + catch (Exception e) + { + log.warn("Could not determine which EventDataIds have attribute values; continuing with the merge.", e); + } + } + @Override public int mergeRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) @@ -207,8 +230,7 @@ public int mergeRows(User user, Container container, DataIteratorBuilder rows, B eventDataIdsByUri.put(objectURI, eventDataId); } - SNDManager.logIds(log, "Attribute values about to be cleared by this merge; the _SND Attribute Data step must re-insert them.", - getEventDataIdsWithAttributeData(container, eventDataIdsByUri)); + logAttributeDataToBeCleared(container, eventDataIdsByUri, log); int count = 0; for(Map map : data) From 74267b5d51bfdf492081862fd1341daa8ed1f3fe Mon Sep 17 00:00:00 2001 From: Marty Pradere Date: Tue, 18 Aug 2026 20:53:01 -0600 Subject: [PATCH 3/6] Name the EventDataId in the skipped-attribute-value log lines These two lines carried the property name but nothing identifying the row, so a single attribute dropped from an otherwise healthy EventDataId could not be joined back to the id sets the rest of this branch logs. Per-attribute loss was invisible to every one of them, since they all reconcile at EventDataId granularity and an id with any resolving value counts as written. --- snd/src/org/labkey/snd/query/AttributeDataTable.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/snd/src/org/labkey/snd/query/AttributeDataTable.java b/snd/src/org/labkey/snd/query/AttributeDataTable.java index 18f4ba029..f666f6a76 100644 --- a/snd/src/org/labkey/snd/query/AttributeDataTable.java +++ b/snd/src/org/labkey/snd/query/AttributeDataTable.java @@ -305,11 +305,11 @@ else if (stringValue != null) { if (pd.getLookupSchema() != null && pd.getLookupQuery() != null) { - logger.info("Value null for property " + pd.getName() + ". Value skipped. Verify lookup " + pd.getLookupSchema() + "." + pd.getLookupQuery() + " contains " + stringValue); + logger.info("Value null for property " + pd.getName() + ", EventDataId: " + eventDataId + ". Value skipped. Verify lookup " + pd.getLookupSchema() + "." + pd.getLookupQuery() + " contains " + stringValue); } else { - logger.info("Value null for property " + pd.getName() + ". Value skipped."); + logger.info("Value null for property " + pd.getName() + ", EventDataId: " + eventDataId + ". Value skipped."); } } From dbd6710eec923c410f394f25cde9c0bdf1f77de7 Mon Sep 17 00:00:00 2001 From: Marty Pradere Date: Thu, 20 Aug 2026 09:22:40 -0600 Subject: [PATCH 4/6] Log EventDataIds on the non-merge paths and drop diagnostics to debug The event data step logged ids only from mergeRows, and only the subset that already carried attribute values, so there was nothing to diff against the id sets the attribute data step logs. importRows and deleteRows now log theirs as well: a newly inserted row the _SND Attribute Data step misses ends up just as stripped as a cleared one, and a deleted row otherwise reads as an attribute data miss. Everything this branch adds now logs at debug, which the ETL job logger runs at by default. The pre-existing Begin/End updating exp.ObjectProperty lines are back at info, and the two diagnostics that cost real work - the exp.Object lookup for values about to be cleared, and the source ordering check - are skipped entirely when debug is off. MAX_LOGGED_IDS drops to 2000, below the 5000 row ETL batch size, so the full batches of an initial load suppress the id lists while the smaller batches of an incremental run keep them. --- snd/src/org/labkey/snd/SNDManager.java | 12 ++++++--- .../labkey/snd/query/AttributeDataTable.java | 18 ++++++++----- .../org/labkey/snd/query/EventDataTable.java | 26 ++++++++++++++++--- 3 files changed, 41 insertions(+), 15 deletions(-) diff --git a/snd/src/org/labkey/snd/SNDManager.java b/snd/src/org/labkey/snd/SNDManager.java index 9a2e43805..eb24300d5 100644 --- a/snd/src/org/labkey/snd/SNDManager.java +++ b/snd/src/org/labkey/snd/SNDManager.java @@ -154,7 +154,8 @@ public static UserSchema getSndUserSchemaAdminRole(Container c, User u) public static int MAX_MERGE_ROWS = 2000; - public static final int MAX_LOGGED_IDS = 5000; + /** Below the ETL batch size so that the full batches of an initial load suppress the id lists while the smaller batches of an incremental run keep them. */ + public static final int MAX_LOGGED_IDS = 2000; private static final int LOGGED_IDS_PER_LINE = 250; public static Logger getLogger(Map configParameters, Class clazz) @@ -175,20 +176,23 @@ public static Logger getLogger(Map configParameters, Class claz */ public static void logIds(Logger log, String message, Collection ids) { - log.info(message + " Count: " + ids.size() + "."); + if (!log.isDebugEnabled()) + return; + + log.debug(message + " Count: " + ids.size() + "."); if (ids.isEmpty()) return; if (ids.size() > MAX_LOGGED_IDS) { - log.info("Id list omitted, more than " + MAX_LOGGED_IDS + " ids."); + log.debug("Id list omitted, more than " + MAX_LOGGED_IDS + " ids."); return; } List sorted = ids.stream().filter(Objects::nonNull).sorted().collect(Collectors.toList()); for (List chunk : ListUtils.partition(sorted, LOGGED_IDS_PER_LINE)) - log.info(" " + StringUtils.join(chunk, ", ")); + log.debug(" " + StringUtils.join(chunk, ", ")); } public static String getPackageName(int id) diff --git a/snd/src/org/labkey/snd/query/AttributeDataTable.java b/snd/src/org/labkey/snd/query/AttributeDataTable.java index f666f6a76..e810cd1d7 100644 --- a/snd/src/org/labkey/snd/query/AttributeDataTable.java +++ b/snd/src/org/labkey/snd/query/AttributeDataTable.java @@ -149,7 +149,7 @@ public QueryUpdateService getUpdateService() protected class UpdateService extends SNDQueryUpdateService { - /** Bounds the source ordering check below. It costs one retained URI per distinct EventDataId, and an ungrouped source would otherwise warn once per row. */ + /** Bounds the source ordering check below. It costs one retained URI per distinct EventDataId, and an ungrouped source would otherwise log once per row. */ private static final int MAX_TRACKED_URIS = 50_000; private static final int MAX_ORDER_WARNINGS = 10; @@ -234,11 +234,13 @@ private int insertObject(Container c, User u, String uri, List p private List> updateObjectProperty(User user, Container container, List> data, boolean isInsertOnly, boolean isUpdate, Logger logger) { + logger.info("Begin updating exp.ObjectProperty."); + Set incomingEventDataIds = new HashSet<>(); for (Map row : data) incomingEventDataIds.add((Integer) row.get("EventDataId")); - SNDManager.logIds(logger, "Begin updating exp.ObjectProperty. Source rows: " + data.size() + ". EventDataIds in this batch:", incomingEventDataIds); + SNDManager.logIds(logger, "Source rows: " + data.size() + ". EventDataIds in this batch:", incomingEventDataIds); int inserted = 0; @@ -250,7 +252,7 @@ private List> updateObjectProperty(User user, Container cont Set cacheEventIds = new HashSet<>(); Set writtenEventDataIds = new HashSet<>(); Set flushedUris = new HashSet<>(); - boolean checkOrdering = true; + boolean checkOrdering = logger.isDebugEnabled(); int outOfOrderFlushes = 0; for(Map row : data) @@ -339,11 +341,11 @@ else if (stringValue != null) if (checkOrdering) { if (!flushedUris.add(prevUri) && ++outOfOrderFlushes <= MAX_ORDER_WARNINGS) - logger.warn("Source rows are not grouped by EventDataId; exp.ObjectProperty for {} was written in more than one pass.", prevUri); + logger.debug("Source rows are not grouped by EventDataId; exp.ObjectProperty for {} was written in more than one pass.", prevUri); if (flushedUris.size() >= MAX_TRACKED_URIS) { - logger.info("More than {} EventDataIds in this batch; ending the source ordering check.", MAX_TRACKED_URIS); + logger.debug("More than {} EventDataIds in this batch; ending the source ordering check.", MAX_TRACKED_URIS); flushedUris.clear(); checkOrdering = false; } @@ -382,10 +384,12 @@ else if (stringValue != null) OntologyManager.clearPropertyCache(); - SNDManager.logIds(logger, "End updating exp.ObjectProperty. Inserted/Updated " + inserted + " rows. EventDataIds written:", writtenEventDataIds); + logger.info("End updating exp.ObjectProperty. Inserted/Updated " + inserted + " rows."); + + SNDManager.logIds(logger, "EventDataIds written:", writtenEventDataIds); if (outOfOrderFlushes > MAX_ORDER_WARNINGS) - logger.warn("{} objectURIs in total were written in more than one pass; further warnings were suppressed.", outOfOrderFlushes); + logger.debug("{} objectURIs in total were written in more than one pass; further messages were suppressed.", outOfOrderFlushes); // Collect only the misses; copying the incoming set would double its footprint on a full load. Set unwritten = new HashSet<>(); diff --git a/snd/src/org/labkey/snd/query/EventDataTable.java b/snd/src/org/labkey/snd/query/EventDataTable.java index dfe340a63..ca832e3b0 100644 --- a/snd/src/org/labkey/snd/query/EventDataTable.java +++ b/snd/src/org/labkey/snd/query/EventDataTable.java @@ -164,9 +164,12 @@ private Set getEventDataIdsWithAttributeData(Container container, Map eventDataIdsByUri, Logger log) { + if (!log.isDebugEnabled()) + return; + if (eventDataIdsByUri.size() > SNDManager.MAX_LOGGED_IDS) { - log.info("More than " + SNDManager.MAX_LOGGED_IDS + " EventDataIds in this batch; skipping the check for attribute values about to be cleared."); + log.debug("More than " + SNDManager.MAX_LOGGED_IDS + " EventDataIds in this batch; skipping the check for attribute values about to be cleared."); return; } @@ -177,7 +180,7 @@ private void logAttributeDataToBeCleared(Container container, Map eventDataIds = new HashSet<>(); for(Map map : data) { - String objectURI = getObjectURI((Integer) map.get("EventDataId"), container); + Integer eventDataId = (Integer) map.get("EventDataId"); + String objectURI = getObjectURI(eventDataId, container); //update snd.EventData row with objectURI map.put("ObjectURI", objectURI); @@ -299,6 +305,8 @@ public int importRows(User user, Container container, DataIteratorBuilder rows, //add to list of cached narrative rows to delete cacheData.add((Integer) map.get("EventId")); + eventDataIds.add(eventDataId); + count++; //TODO: Count in exp.Object is not going to be the same as in snd.EventData - need to figure out how to get the count to log if(count % 1000 == 0) @@ -306,6 +314,9 @@ public int importRows(User user, Container container, DataIteratorBuilder rows, } log.info("End inserting into exp.Object. Inserted total of " + count + " rows."); + // These rows get a fresh exp.Object with no properties, so they depend on the _SND Attribute Data step just as much as the merged ones do. + SNDManager.logIds(log, "EventDataIds inserted into snd.EventData by this batch:", eventDataIds); + DataIteratorBuilder rowsWithObjectURI = new ListofMapsDataIterator.Builder(data.get(0).keySet(), data); _sndManager.updateNarrativeCache(container, user, cacheData, log); @@ -405,11 +416,15 @@ private void deleteFromExpTables(List> oldRows, Container co { log.info("Begin deleting from exp.ObjectProperty and exp.Object."); int count = 0; + Set eventDataIds = new HashSet<>(); //This will be a cascading delete across exp.ObjectProperty, exp.Object, and snd.EventData for (Map map : oldRows) { - String objectURI = getObjectURI((Integer) map.get("EventDataId"), container); + Integer eventDataId = (Integer) map.get("EventDataId"); + String objectURI = getObjectURI(eventDataId, container); + + eventDataIds.add(eventDataId); OntologyObject obj = OntologyManager.getOntologyObject(container, objectURI); //delete row from exp.ObjectProperty @@ -426,6 +441,9 @@ private void deleteFromExpTables(List> oldRows, Container co } log.info("End deleting from exp.ObjectProperty and exp.Object. Deleted total of " + count + " rows."); + + // Without these the deleted rows read as attribute data the _SND Attribute Data step failed to write. + SNDManager.logIds(log, "EventDataIds deleted from snd.EventData by this batch:", eventDataIds); } private int deleteAllFromExpTables(Logger log) From 7c7c50009bc6a8413ac69b42900093ec8cb8af57 Mon Sep 17 00:00:00 2001 From: Marty Pradere Date: Sun, 23 Aug 2026 14:21:51 -0600 Subject: [PATCH 5/6] Log the source rowversion span of each SND ETL batch The two source views derive their rowversion differently - v_snd_eventData from CODED_PROCS alone, v_snd_attributeData from MAX over CODED_PROCS and CODED_PROC_ATTRIBS - and each step clamps its own window end to MIN_ACTIVE_ROWVERSION at the moment it runs, so the two incremental windows can cover different rows even though the values sit on one sequence. Logging the span a batch actually covered puts that next to the window the ETL already logs. The attribute step also pairs each EventDataId it left unwritten with its rowversion, on a line of its own so the bare id list stays diffable against the event step's. An id whose rowversion falls inside the window was not excluded by it: it is missing from v_snd_attributeData altogether, which the inner joins on CODED_PROC_ATTRIBS and PKG_ATTRIBS and the blank-value filter can each cause, and which does not heal on the next run. The delete path gets no span; its source view filters on audit_date_tm rather than the rowversion column. --- snd/src/org/labkey/snd/SNDManager.java | 61 +++++++++++++++++++ .../labkey/snd/query/AttributeDataTable.java | 19 +++++- .../org/labkey/snd/query/EventDataTable.java | 2 + 3 files changed, 81 insertions(+), 1 deletion(-) diff --git a/snd/src/org/labkey/snd/SNDManager.java b/snd/src/org/labkey/snd/SNDManager.java index eb24300d5..7826ce291 100644 --- a/snd/src/org/labkey/snd/SNDManager.java +++ b/snd/src/org/labkey/snd/SNDManager.java @@ -91,6 +91,7 @@ import org.labkey.snd.security.SNDSecurityManager; import org.labkey.snd.trigger.SNDTriggerManager; +import java.nio.ByteBuffer; import java.sql.SQLException; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -104,6 +105,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.LongSummaryStatistics; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -157,6 +159,10 @@ public static UserSchema getSndUserSchemaAdminRole(Container c, User u) /** Below the ETL batch size so that the full batches of an initial load suppress the id lists while the smaller batches of an incremental run keep them. */ public static final int MAX_LOGGED_IDS = 2000; private static final int LOGGED_IDS_PER_LINE = 250; + private static final int LOGGED_PAIRS_PER_LINE = 50; + + /** The incremental filter column of both SND ETL source views. */ + public static final String SOURCE_ROWVERSION_COLUMN = "timestamp"; public static Logger getLogger(Map configParameters, Class clazz) { @@ -195,6 +201,61 @@ public static void logIds(Logger log, String message, Collection ids) log.debug(" " + StringUtils.join(chunk, ", ")); } + /** + * SQL Server hands a rowversion back as binary(8); the ETL's own persisted window state returns it as a number. + * Read big-endian, matching how the incremental filter logs its bounds, so the two can be compared directly. + */ + @Nullable + public static Long toRowversion(@Nullable Object o) + { + if (o instanceof byte[] bytes && 8 == bytes.length) + return ByteBuffer.wrap(bytes).getLong(); + if (o instanceof Number n) + return n.longValue(); + return null; + } + + /** + * Logs the rowversion span of a batch so it can be placed against the incremental window the ETL logged for the + * run. Both SND source views draw their rowversions from the same source database, so the spans the two steps + * report are on one sequence and comparable. + */ + public static void logRowversionRange(Logger log, String message, Collection> rows) + { + if (!log.isDebugEnabled()) + return; + + LongSummaryStatistics stats = rows.stream() + .map(row -> toRowversion(row.get(SOURCE_ROWVERSION_COLUMN))) + .filter(Objects::nonNull) + .mapToLong(Long::longValue) + .summaryStatistics(); + + if (0 == stats.getCount()) + log.debug(message + " No source rowversions in this batch."); + else + log.debug(message + " Rowversions " + stats.getMin() + " to " + stats.getMax() + " over " + stats.getCount() + " rows."); + } + + /** + * Pairs each id with its source rowversion. Logged separately from the bare list the same set gets from logIds, + * which stays free of annotations so it can be diffed against the other step's list. + */ + public static void logIdRowversions(Logger log, String message, Collection ids, Map rowversions) + { + if (!log.isDebugEnabled() || ids.isEmpty() || ids.size() > MAX_LOGGED_IDS) + return; + + log.debug(message); + + List pairs = ids.stream().filter(Objects::nonNull).sorted() + .map(id -> id + ":" + rowversions.get(id)) + .collect(Collectors.toList()); + + for (List chunk : ListUtils.partition(pairs, LOGGED_PAIRS_PER_LINE)) + log.debug(" " + StringUtils.join(chunk, ", ")); + } + public static String getPackageName(int id) { return PackageDomainKind.getPackageKindName() + "-" + id; diff --git a/snd/src/org/labkey/snd/query/AttributeDataTable.java b/snd/src/org/labkey/snd/query/AttributeDataTable.java index e810cd1d7..cb4a490d8 100644 --- a/snd/src/org/labkey/snd/query/AttributeDataTable.java +++ b/snd/src/org/labkey/snd/query/AttributeDataTable.java @@ -236,11 +236,25 @@ private List> updateObjectProperty(User user, Container cont { logger.info("Begin updating exp.ObjectProperty."); + // An EventDataId gets one source row per attribute; keep the newest, since that is the one that pulled it into the window. + boolean trackRowversions = logger.isDebugEnabled(); Set incomingEventDataIds = new HashSet<>(); + Map rowversionByEventDataId = new HashMap<>(); for (Map row : data) - incomingEventDataIds.add((Integer) row.get("EventDataId")); + { + Integer eventDataId = (Integer) row.get("EventDataId"); + incomingEventDataIds.add(eventDataId); + + if (trackRowversions) + { + Long rowversion = SNDManager.toRowversion(row.get(SNDManager.SOURCE_ROWVERSION_COLUMN)); + if (null != rowversion) + rowversionByEventDataId.merge(eventDataId, rowversion, Math::max); + } + } SNDManager.logIds(logger, "Source rows: " + data.size() + ". EventDataIds in this batch:", incomingEventDataIds); + SNDManager.logRowversionRange(logger, "Source span of this batch.", data); int inserted = 0; @@ -400,7 +414,10 @@ else if (stringValue != null) } if (!unwritten.isEmpty()) + { SNDManager.logIds(logger, "EventDataIds present in the source rows but left with no attribute values written:", unwritten); + SNDManager.logIdRowversions(logger, "Rowversions of those EventDataIds, to place them against the incremental window of this run:", unwritten, rowversionByEventDataId); + } _sndManager.updateNarrativeCache(container, user, cacheEventIds, logger); diff --git a/snd/src/org/labkey/snd/query/EventDataTable.java b/snd/src/org/labkey/snd/query/EventDataTable.java index ca832e3b0..57491ebd9 100644 --- a/snd/src/org/labkey/snd/query/EventDataTable.java +++ b/snd/src/org/labkey/snd/query/EventDataTable.java @@ -234,6 +234,7 @@ public int mergeRows(User user, Container container, DataIteratorBuilder rows, B } SNDManager.logIds(log, "EventDataIds merged into snd.EventData by this batch:", eventDataIdsByUri.values()); + SNDManager.logRowversionRange(log, "Source span of the merged rows.", data); logAttributeDataToBeCleared(container, eventDataIdsByUri, log); int count = 0; @@ -316,6 +317,7 @@ public int importRows(User user, Container container, DataIteratorBuilder rows, // These rows get a fresh exp.Object with no properties, so they depend on the _SND Attribute Data step just as much as the merged ones do. SNDManager.logIds(log, "EventDataIds inserted into snd.EventData by this batch:", eventDataIds); + SNDManager.logRowversionRange(log, "Source span of the inserted rows.", data); DataIteratorBuilder rowsWithObjectURI = new ListofMapsDataIterator.Builder(data.get(0).keySet(), data); From 0ba677cb1e1ec8c86e9c8f3a047429f601999408 Mon Sep 17 00:00:00 2001 From: Marty Pradere Date: Fri, 28 Aug 2026 22:22:17 -0600 Subject: [PATCH 6/6] Log SND ETL rowversions through BIGINT copies of the source column The rowversion span logged by the previous commit never appeared: TransformDataIteratorBuilder drops every source column SQL Server types as a rowversion, so the incremental filter column itself never reaches the steps that log it and every batch reported no rowversions. Both source views now expose their rowversions a second time as BIGINT, which passes through untouched. The filter columns are unchanged, so the incremental windows are unaffected. The attribute data view exposes the coded proc row's and the attribute row's rowversions separately rather than only the MAX it filters on, and each unwritten EventDataId is now marked with the side its rowversion came from. An (a) id is newer on the attribute row than on the coded proc row the event step filtered on, so this step's window can exclude an id that step just cleared and the next run restores it; a (p) id carries the value the event step saw, so the window is not what kept it out and no later run brings it back. The views deploy separately from the module, so until they are updated both helpers report that the source view is missing the BIGINT copies rather than logging a column of nulls. --- snd/src/org/labkey/snd/SNDManager.java | 85 ++++++++++++++----- .../labkey/snd/query/AttributeDataTable.java | 14 +-- .../org/labkey/snd/query/EventDataTable.java | 4 +- .../create_v_snd_attributeData.sql | 8 +- .../source_queries/create_v_snd_eventData.sql | 5 +- 5 files changed, 85 insertions(+), 31 deletions(-) diff --git a/snd/src/org/labkey/snd/SNDManager.java b/snd/src/org/labkey/snd/SNDManager.java index 7826ce291..9680ee99e 100644 --- a/snd/src/org/labkey/snd/SNDManager.java +++ b/snd/src/org/labkey/snd/SNDManager.java @@ -86,17 +86,18 @@ import org.labkey.api.snd.SuperPackage; import org.labkey.api.util.DateUtil; import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.StringUtilsLabKey; import org.labkey.snd.query.PackagesTable; import org.labkey.snd.security.QCStateActionEnum; import org.labkey.snd.security.SNDSecurityManager; import org.labkey.snd.trigger.SNDTriggerManager; -import java.nio.ByteBuffer; import java.sql.SQLException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; @@ -161,8 +162,15 @@ public static UserSchema getSndUserSchemaAdminRole(Container c, User u) private static final int LOGGED_IDS_PER_LINE = 250; private static final int LOGGED_PAIRS_PER_LINE = 50; - /** The incremental filter column of both SND ETL source views. */ - public static final String SOURCE_ROWVERSION_COLUMN = "timestamp"; + /** + * The BIGINT copy each SND ETL source view exposes of the coded proc row's rowversion. The ETL drops every column + * SQL Server types as a rowversion (TransformDataIteratorBuilder, via TransformManager.isRowversionColumn), so the + * incremental filter column itself never reaches these steps and only a cast copy of it can be logged. + */ + public static final String PROC_ROWVERSION_COLUMN = "ProcRowversion"; + + /** The same for the attribute row's rowversion, which only v_snd_attributeData has. */ + public static final String ATTRIB_ROWVERSION_COLUMN = "AttribRowversion"; public static Logger getLogger(Map configParameters, Class clazz) { @@ -201,52 +209,85 @@ public static void logIds(Logger log, String message, Collection ids) log.debug(" " + StringUtils.join(chunk, ", ")); } - /** - * SQL Server hands a rowversion back as binary(8); the ETL's own persisted window state returns it as a number. - * Read big-endian, matching how the incremental filter logs its bounds, so the two can be compared directly. - */ + /** The cast copies arrive as BIGINT, the form the incremental filter also logs its bounds in. */ @Nullable public static Long toRowversion(@Nullable Object o) { - if (o instanceof byte[] bytes && 8 == bytes.length) - return ByteBuffer.wrap(bytes).getLong(); - if (o instanceof Number n) - return n.longValue(); - return null; + return o instanceof Number n ? n.longValue() : null; + } + + /** + * A source row's rowversion and the source table it came from. v_snd_attributeData filters on the newer of the + * coded proc row and the attribute row, so a row can enter that step's window on a value the event step, which + * sees only the coded proc row, never had. + */ + public record SourceRowversion(long value, boolean fromAttribute) + { + /** Null when the row carries neither column, which is what a view still lacking the cast copies looks like. */ + @Nullable + public static SourceRowversion of(@Nullable Long proc, @Nullable Long attrib) + { + if (null == proc) + return null == attrib ? null : new SourceRowversion(attrib, true); + return null == attrib || attrib <= proc ? new SourceRowversion(proc, false) : new SourceRowversion(attrib, true); + } + + public static SourceRowversion later(SourceRowversion a, SourceRowversion b) + { + return b.value > a.value ? b : a; + } + + @Override + public String toString() + { + return value + (fromAttribute ? "(a)" : "(p)"); + } } /** * Logs the rowversion span of a batch so it can be placed against the incremental window the ETL logged for the * run. Both SND source views draw their rowversions from the same source database, so the spans the two steps - * report are on one sequence and comparable. + * report are on one sequence and comparable. Pass every column the view derives its filter value from; the span is + * over the newest of them per row. */ - public static void logRowversionRange(Logger log, String message, Collection> rows) + public static void logRowversionRange(Logger log, String message, Collection> rows, String... columns) { if (!log.isDebugEnabled()) return; LongSummaryStatistics stats = rows.stream() - .map(row -> toRowversion(row.get(SOURCE_ROWVERSION_COLUMN))) + .map(row -> Arrays.stream(columns).map(column -> toRowversion(row.get(column))).filter(Objects::nonNull).max(Long::compare).orElse(null)) .filter(Objects::nonNull) .mapToLong(Long::longValue) .summaryStatistics(); - if (0 == stats.getCount()) - log.debug(message + " No source rowversions in this batch."); + if (rows.isEmpty()) + log.debug(message + " Empty batch."); + else if (0 == stats.getCount()) + log.debug(message + " No source rowversions in " + StringUtilsLabKey.pluralize(rows.size(), "row") + "; the source view is missing the BIGINT rowversion copies."); else - log.debug(message + " Rowversions " + stats.getMin() + " to " + stats.getMax() + " over " + stats.getCount() + " rows."); + log.debug(message + " Rowversions " + stats.getMin() + " to " + stats.getMax() + " over " + StringUtilsLabKey.pluralize(stats.getCount(), "row") + "."); } /** - * Pairs each id with its source rowversion. Logged separately from the bare list the same set gets from logIds, - * which stays free of annotations so it can be diffed against the other step's list. + * Pairs each id with its source rowversion and the side that rowversion came from. Logged separately from the bare + * list the same set gets from logIds, which stays free of annotations so it can be diffed against the other step's + * list. */ - public static void logIdRowversions(Logger log, String message, Collection ids, Map rowversions) + public static void logIdRowversions(Logger log, String message, Collection ids, Map rowversions) { if (!log.isDebugEnabled() || ids.isEmpty() || ids.size() > MAX_LOGGED_IDS) return; - log.debug(message); + List known = ids.stream().map(rowversions::get).filter(Objects::nonNull).collect(Collectors.toList()); + if (known.isEmpty()) + { + log.debug(message + " No source rowversions; the source view is missing the BIGINT rowversion copies."); + return; + } + + long fromAttribute = known.stream().filter(SourceRowversion::fromAttribute).count(); + log.debug(message + " " + fromAttribute + " of " + StringUtilsLabKey.pluralize(known.size(), "id") + " with a rowversion took it from the attribute row (a), the rest from the coded proc row (p)."); List pairs = ids.stream().filter(Objects::nonNull).sorted() .map(id -> id + ":" + rowversions.get(id)) diff --git a/snd/src/org/labkey/snd/query/AttributeDataTable.java b/snd/src/org/labkey/snd/query/AttributeDataTable.java index cb4a490d8..1e675151a 100644 --- a/snd/src/org/labkey/snd/query/AttributeDataTable.java +++ b/snd/src/org/labkey/snd/query/AttributeDataTable.java @@ -239,7 +239,7 @@ private List> updateObjectProperty(User user, Container cont // An EventDataId gets one source row per attribute; keep the newest, since that is the one that pulled it into the window. boolean trackRowversions = logger.isDebugEnabled(); Set incomingEventDataIds = new HashSet<>(); - Map rowversionByEventDataId = new HashMap<>(); + Map rowversionByEventDataId = new HashMap<>(); for (Map row : data) { Integer eventDataId = (Integer) row.get("EventDataId"); @@ -247,14 +247,16 @@ private List> updateObjectProperty(User user, Container cont if (trackRowversions) { - Long rowversion = SNDManager.toRowversion(row.get(SNDManager.SOURCE_ROWVERSION_COLUMN)); + SNDManager.SourceRowversion rowversion = SNDManager.SourceRowversion.of( + SNDManager.toRowversion(row.get(SNDManager.PROC_ROWVERSION_COLUMN)), + SNDManager.toRowversion(row.get(SNDManager.ATTRIB_ROWVERSION_COLUMN))); if (null != rowversion) - rowversionByEventDataId.merge(eventDataId, rowversion, Math::max); + rowversionByEventDataId.merge(eventDataId, rowversion, SNDManager.SourceRowversion::later); } } SNDManager.logIds(logger, "Source rows: " + data.size() + ". EventDataIds in this batch:", incomingEventDataIds); - SNDManager.logRowversionRange(logger, "Source span of this batch.", data); + SNDManager.logRowversionRange(logger, "Source span of this batch.", data, SNDManager.PROC_ROWVERSION_COLUMN, SNDManager.ATTRIB_ROWVERSION_COLUMN); int inserted = 0; @@ -416,7 +418,9 @@ else if (stringValue != null) if (!unwritten.isEmpty()) { SNDManager.logIds(logger, "EventDataIds present in the source rows but left with no attribute values written:", unwritten); - SNDManager.logIdRowversions(logger, "Rowversions of those EventDataIds, to place them against the incremental window of this run:", unwritten, rowversionByEventDataId); + // An (a) id is newer on the attribute row than on the coded proc row the event step filtered on, so this step's window can exclude an id that step just cleared, and the next run restores it. + // A (p) id carries the same value the event step saw, so the window is not what kept it out: it is missing from v_snd_attributeData itself, and no later run brings it back. + SNDManager.logIdRowversions(logger, "Rowversions of those EventDataIds, to place them against the incremental window of this run.", unwritten, rowversionByEventDataId); } _sndManager.updateNarrativeCache(container, user, cacheEventIds, logger); diff --git a/snd/src/org/labkey/snd/query/EventDataTable.java b/snd/src/org/labkey/snd/query/EventDataTable.java index 57491ebd9..d26a3e350 100644 --- a/snd/src/org/labkey/snd/query/EventDataTable.java +++ b/snd/src/org/labkey/snd/query/EventDataTable.java @@ -234,7 +234,7 @@ public int mergeRows(User user, Container container, DataIteratorBuilder rows, B } SNDManager.logIds(log, "EventDataIds merged into snd.EventData by this batch:", eventDataIdsByUri.values()); - SNDManager.logRowversionRange(log, "Source span of the merged rows.", data); + SNDManager.logRowversionRange(log, "Source span of the merged rows.", data, SNDManager.PROC_ROWVERSION_COLUMN); logAttributeDataToBeCleared(container, eventDataIdsByUri, log); int count = 0; @@ -317,7 +317,7 @@ public int importRows(User user, Container container, DataIteratorBuilder rows, // These rows get a fresh exp.Object with no properties, so they depend on the _SND Attribute Data step just as much as the merged ones do. SNDManager.logIds(log, "EventDataIds inserted into snd.EventData by this batch:", eventDataIds); - SNDManager.logRowversionRange(log, "Source span of the inserted rows.", data); + SNDManager.logRowversionRange(log, "Source span of the inserted rows.", data, SNDManager.PROC_ROWVERSION_COLUMN); DataIteratorBuilder rowsWithObjectURI = new ListofMapsDataIterator.Builder(data.get(0).keySet(), data); diff --git a/snprc_ehr/resources/source_queries/create_v_snd_attributeData.sql b/snprc_ehr/resources/source_queries/create_v_snd_attributeData.sql index 89d2c2861..5886e528f 100644 --- a/snprc_ehr/resources/source_queries/create_v_snd_attributeData.sql +++ b/snprc_ehr/resources/source_queries/create_v_snd_attributeData.sql @@ -36,6 +36,7 @@ AS -- Purpose is to handle numeric data with commas (',') ~line 59 srr -- 04/23/2024 Lookup values need to be string values by default. tjh -- 6/26/2025 Added check for eventId in labkey Events table. tjh +-- 8/28/2026 Added ProcRowversion and AttribRowversion so the ETL step can log the rowversion it filtered on and which row it came from. -- ========================================================================================== SELECT TOP (99.999999999) PERCENT cp.ANIMAL_EVENT_ID AS EventId, @@ -57,7 +58,12 @@ SELECT TOP (99.999999999) PERCENT CASE WHEN ( (LOWER(pa.DATA_TYPE)) = 'string' OR pa.LOOKUP_KEY IS NOT NULL) THEN 's' ELSE 'f' END AS TypeTag, cp.OBJECT_ID AS objectId, -( SELECT MAX(v) FROM ( VALUES (cp.timestamp), (cpa.timestamp)) AS VALUE (v)) AS TIMESTAMP +( SELECT MAX(v) FROM ( VALUES (cp.timestamp), (cpa.timestamp)) AS VALUE (v)) AS TIMESTAMP, + +-- The ETL drops every column SQL Server types as a rowversion, so its steps can only see these cast copies. TIMESTAMP above stays the incremental filter column. +-- Kept apart rather than pre-maxed: an EventDataId written with no attribute values is diagnosed differently depending on which of the two rows carried its rowversion. +CAST(cp.timestamp AS BIGINT) AS ProcRowversion, +CAST(cpa.timestamp AS BIGINT) AS AttribRowversion FROM dbo.CODED_PROCS AS cp diff --git a/snprc_ehr/resources/source_queries/create_v_snd_eventData.sql b/snprc_ehr/resources/source_queries/create_v_snd_eventData.sql index c3e29f4e7..4ab6c36a5 100644 --- a/snprc_ehr/resources/source_queries/create_v_snd_eventData.sql +++ b/snprc_ehr/resources/source_queries/create_v_snd_eventData.sql @@ -21,6 +21,7 @@ AS -- Description: View provides the datasource for event data with attribute/values -- Changes: 4/12/2018 Added permissions -- 6/26/2025 Added check for eventId in labkey Events table. tjh + -- 8/28/2026 Added ProcRowversion so the ETL step can log the rowversion it filtered on. -- ========================================================================================== SELECT @@ -29,7 +30,9 @@ AS cp.PROC_ID AS EventDataId, cp.PARENT_PROC_ID AS ParentEventDataId, sp.SUPER_PKG_ID AS SuperPkgId, - cp.timestamp AS timestamp + cp.timestamp AS timestamp, + -- The ETL drops every column SQL Server types as a rowversion, so its steps can only see this cast copy. [timestamp] above stays the incremental filter column. + CAST(cp.timestamp AS BIGINT) AS ProcRowversion FROM dbo.CODED_PROCS AS cp INNER JOIN dbo.ANIMAL_EVENTS AS ae ON cp.ANIMAL_EVENT_ID = ae.ANIMAL_EVENT_ID