Dagster Orchestration

orchestration/schedules.py

@@ -172,8 +172,13 @@ def batch_upload_sensor(context: SensorEvaluationContext):
     NOTE ON THE NORMAL PRODUCTION PATH: nothing here is involved. The API runs
     a staged batch itself, on a bounded worker thread, and this sensor exists
     for the development machine where Dagster is the thing driving the work.
-    Turning it on alongside a running API would mean both trying to ingest the
-    same batch, which is why it ships STOPPED.
+
+    It is now safe to run alongside a live API. Both executors read the same
+    directory, but a batch carries a `runner` naming which one owns it, and
+    this sensor claims only `runner == "dagster"` - the batches an admin sent
+    here from the orchestration tab. It still ships STOPPED, because a
+    development machine should not start ingesting because a directory
+    happened to have something in it.
     """
     from app.core import batch_ingest
 
@@ -181,12 +186,15 @@ def batch_upload_sensor(context: SensorEvaluationContext):
     if not root.exists():
         return SkipReason("Batch upload directory {} does not exist.".format(root))
 
+    # Only batches an admin explicitly handed to Dagster. See the same filter,
+    # and the reason for it, in assets/batch_catalog.py:_pick_batch_id.
     waiting = [
         m for m in batch_ingest.list_manifests()
         if m.status in (batch_ingest.QUEUED, batch_ingest.INTERRUPTED)
+        and m.runner == batch_ingest.RUNNER_DAGSTER
     ]
     if not waiting:
-        return SkipReason("No staged batch is waiting to run.")
+        return SkipReason("No staged batch is waiting for Dagster.")
 
     already = set(filter(None, (context.cursor or "").split(",")))
     fresh = [m for m in waiting if m.batch_id not in already]