Skip to content

Commit 0856c22

Browse files
authored
Side Input improvements (#38363)
* side-input improvements * remove files * add Override * foo * foo * fix splittable dofn * fix * foo * fix windows * fix compilation * foo
1 parent fe7dd9e commit 0856c22

22 files changed

Lines changed: 2181 additions & 541 deletions

File tree

runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -870,8 +870,17 @@ public InputT element(DoFn<InputT, OutputT> doFn) {
870870
}
871871

872872
@Override
873-
public Object sideInput(String tagId) {
874-
throw new UnsupportedOperationException("SideInput parameters are not supported.");
873+
public @Nullable Object sideInput(String tagId) {
874+
PCollectionView<?> view =
875+
checkStateNotNull(sideInputMapping.get(tagId), "Side input tag %s not found", tagId);
876+
return sideInput(view);
877+
}
878+
879+
@Override
880+
public <T> T sideInput(PCollectionView<T> view) {
881+
checkNotNull(view, "View passed to sideInput cannot be null");
882+
return SimpleDoFnRunner.this.sideInput(
883+
view, view.getWindowMappingFn().getSideInputWindow(window()));
875884
}
876885

877886
@Override
@@ -1196,8 +1205,17 @@ public InputT element(DoFn<InputT, OutputT> doFn) {
11961205
}
11971206

11981207
@Override
1199-
public Object sideInput(String tagId) {
1200-
throw new UnsupportedOperationException("SideInput parameters are not supported.");
1208+
public @Nullable Object sideInput(String tagId) {
1209+
PCollectionView<?> view =
1210+
checkStateNotNull(sideInputMapping.get(tagId), "Side input tag %s not found", tagId);
1211+
return sideInput(view);
1212+
}
1213+
1214+
@Override
1215+
public <T> T sideInput(PCollectionView<T> view) {
1216+
checkNotNull(view, "View passed to sideInput cannot be null");
1217+
return SimpleDoFnRunner.this.sideInput(
1218+
view, view.getWindowMappingFn().getSideInputWindow(window()));
12011219
}
12021220

12031221
@Override

runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/SimpleDoFnRunnerFactory.java

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
import org.apache.beam.runners.core.SideInputReader;
2525
import org.apache.beam.sdk.coders.Coder;
2626
import org.apache.beam.sdk.options.PipelineOptions;
27-
import org.apache.beam.sdk.options.StreamingOptions;
2827
import org.apache.beam.sdk.transforms.DoFn;
2928
import org.apache.beam.sdk.transforms.DoFnSchemaInformation;
3029
import org.apache.beam.sdk.util.WindowedValueMultiReceiver;
@@ -68,17 +67,6 @@ public DoFnRunner<InputT, OutputT> createRunner(
6867
windowingStrategy,
6968
doFnSchemaInformation,
7069
sideInputMapping);
71-
boolean hasStreamingSideInput =
72-
options.as(StreamingOptions.class).isStreaming() && !sideInputReader.isEmpty();
73-
if (hasStreamingSideInput) {
74-
return new StreamingSideInputDoFnRunner<>(
75-
fnRunner,
76-
new StreamingSideInputFetcher<>(
77-
sideInputViews,
78-
inputCoder,
79-
windowingStrategy,
80-
(StreamingModeExecutionContext.StreamingModeStepContext) userStepContext));
81-
}
8270
return fnRunner;
8371
}
8472
}

0 commit comments

Comments
 (0)