From 0f42e485d41d3a03323fab89090d8af083f39b43 Mon Sep 17 00:00:00 2001 From: ebeltran-g Date: Mon, 1 Dec 2025 10:58:42 -0600 Subject: [PATCH 1/4] Adding Dataflow MDC Sample --- .../java/com/example/dataflow/MdcSample.java | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 dataflow/snippets/src/main/java/com/example/dataflow/MdcSample.java diff --git a/dataflow/snippets/src/main/java/com/example/dataflow/MdcSample.java b/dataflow/snippets/src/main/java/com/example/dataflow/MdcSample.java new file mode 100644 index 00000000000..8373ce0af83 --- /dev/null +++ b/dataflow/snippets/src/main/java/com/example/dataflow/MdcSample.java @@ -0,0 +1,83 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.dataflow; + +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.io.gcp.pubsub.PubsubMessage; +import org.apache.beam.sdk.io.gcp.pubsub.PubsubIO; +import org.apache.beam.sdk.options.Description; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.options.SdkHarnessOptions; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.ParDo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; + +public class MdcSample { + + public interface MdcSampleJobOptions extends SdkHarnessOptions { + @Description("The Pub/Sub subscription to read from.") + String getInputSubscription(); + + void setInputSubscription(String value); + } + + public static class MessageReaderFn extends DoFn { + + private transient Logger logger; + + @Setup + public void setup() { + logger = LoggerFactory.getLogger(MessageReaderFn.class); + } + + @ProcessElement + public void processElement(ProcessContext c) { + PubsubMessage message = c.element(); + String messageId = message.getMessageId(); + + try (MDC.MDCCloseable ignored = MDC.putCloseable("messageId", messageId)) { + String payload = new String(message.getPayload()); + logger.info("Received message with payload: " + payload); + + // This is the example task + logger.info("Executing example task..."); + } catch (Exception e) { + logger.info("failure"); + } + } + } + + public static void main(String[] args) { + MdcSampleJobOptions options = + PipelineOptionsFactory.fromArgs(args).withValidation().as(MdcSampleJobOptions.class); + // options.setRunner(DirectRunner.class); + + options.setLogMdc(true); + + + Pipeline p = Pipeline.create(options); + + p.apply( + "Read Messages from Pub/Sub", + PubsubIO.readMessagesWithAttributes().fromSubscription(options.getInputSubscription())) + .apply("Process Message", ParDo.of(new MessageReaderFn())); + + p.run(); + } +} \ No newline at end of file From c05701bc781baee004cf54fe7a6a0eca23f60dac Mon Sep 17 00:00:00 2001 From: ebeltran-g <95712847+ebeltran-g@users.noreply.github.com> Date: Mon, 1 Dec 2025 11:12:26 -0600 Subject: [PATCH 2/4] Update dataflow/snippets/src/main/java/com/example/dataflow/MdcSample.java Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../snippets/src/main/java/com/example/dataflow/MdcSample.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataflow/snippets/src/main/java/com/example/dataflow/MdcSample.java b/dataflow/snippets/src/main/java/com/example/dataflow/MdcSample.java index 8373ce0af83..3752051ebc9 100644 --- a/dataflow/snippets/src/main/java/com/example/dataflow/MdcSample.java +++ b/dataflow/snippets/src/main/java/com/example/dataflow/MdcSample.java @@ -58,7 +58,7 @@ public void processElement(ProcessContext c) { // This is the example task logger.info("Executing example task..."); } catch (Exception e) { - logger.info("failure"); + logger.error("Failed to process message", e); } } } From e2ee79039a2606ee3b1f5da264fa44abe6ad3083 Mon Sep 17 00:00:00 2001 From: ebeltran-g <95712847+ebeltran-g@users.noreply.github.com> Date: Mon, 1 Dec 2025 11:13:24 -0600 Subject: [PATCH 3/4] Update dataflow/snippets/src/main/java/com/example/dataflow/MdcSample.java Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../snippets/src/main/java/com/example/dataflow/MdcSample.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataflow/snippets/src/main/java/com/example/dataflow/MdcSample.java b/dataflow/snippets/src/main/java/com/example/dataflow/MdcSample.java index 3752051ebc9..63c8eee5625 100644 --- a/dataflow/snippets/src/main/java/com/example/dataflow/MdcSample.java +++ b/dataflow/snippets/src/main/java/com/example/dataflow/MdcSample.java @@ -53,7 +53,7 @@ public void processElement(ProcessContext c) { try (MDC.MDCCloseable ignored = MDC.putCloseable("messageId", messageId)) { String payload = new String(message.getPayload()); - logger.info("Received message with payload: " + payload); + logger.info("Received message with payload: {}", payload); // This is the example task logger.info("Executing example task..."); From bea9306d3d951abb8500d23383f1446bbbc7e3d8 Mon Sep 17 00:00:00 2001 From: ebeltran-g <95712847+ebeltran-g@users.noreply.github.com> Date: Mon, 1 Dec 2025 11:13:35 -0600 Subject: [PATCH 4/4] Update dataflow/snippets/src/main/java/com/example/dataflow/MdcSample.java Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../snippets/src/main/java/com/example/dataflow/MdcSample.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataflow/snippets/src/main/java/com/example/dataflow/MdcSample.java b/dataflow/snippets/src/main/java/com/example/dataflow/MdcSample.java index 63c8eee5625..a2cf3da3de4 100644 --- a/dataflow/snippets/src/main/java/com/example/dataflow/MdcSample.java +++ b/dataflow/snippets/src/main/java/com/example/dataflow/MdcSample.java @@ -52,7 +52,7 @@ public void processElement(ProcessContext c) { String messageId = message.getMessageId(); try (MDC.MDCCloseable ignored = MDC.putCloseable("messageId", messageId)) { - String payload = new String(message.getPayload()); + String payload = new String(message.getPayload(), java.nio.charset.StandardCharsets.UTF_8); logger.info("Received message with payload: {}", payload); // This is the example task