diff --git a/genai/snippets/resources/output/bounding-boxes-socks.jpg b/genai/snippets/resources/output/bounding-boxes-socks.jpg new file mode 100644 index 00000000000..dbb91b49076 Binary files /dev/null and b/genai/snippets/resources/output/bounding-boxes-socks.jpg differ diff --git a/genai/snippets/src/main/java/genai/boundingbox/BoundingBoxWithTxtImg.java b/genai/snippets/src/main/java/genai/boundingbox/BoundingBoxWithTxtImg.java new file mode 100644 index 00000000000..8988c7c8d01 --- /dev/null +++ b/genai/snippets/src/main/java/genai/boundingbox/BoundingBoxWithTxtImg.java @@ -0,0 +1,217 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package genai.boundingbox; + +// [START googlegenaisdk_boundingbox_with_txt_img] + +import static com.google.genai.types.Type.Known.ARRAY; +import static com.google.genai.types.Type.Known.INTEGER; +import static com.google.genai.types.Type.Known.OBJECT; +import static com.google.genai.types.Type.Known.STRING; + +import com.google.genai.Client; +import com.google.genai.types.Content; +import com.google.genai.types.GenerateContentConfig; +import com.google.genai.types.GenerateContentResponse; +import com.google.genai.types.HarmBlockThreshold; +import com.google.genai.types.HarmCategory; +import com.google.genai.types.HttpOptions; +import com.google.genai.types.Part; +import com.google.genai.types.SafetySetting; +import com.google.genai.types.Schema; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import java.awt.BasicStroke; +import java.awt.Color; +import java.awt.Font; +import java.awt.Graphics2D; +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Type; +import java.net.URL; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import javax.imageio.ImageIO; + +public class BoundingBoxWithTxtImg { + + public static class BoundingBox { + List box2d; + String label; + + public List getBox2d() { + return box2d; + } + + public String getLabel() { + return label; + } + } + + // Plot bounding boxes on an image and save it to a file. + public static void plotBoundingBoxes(String imageUrl, List boundingBoxes) + throws IOException { + URL url = new URL(imageUrl); + BufferedImage image = ImageIO.read(url); + + int width = image.getWidth(); + int height = image.getHeight(); + + Graphics2D graphics2D = image.createGraphics(); + graphics2D.setStroke(new BasicStroke(4)); + graphics2D.setFont(new Font("Arial", Font.PLAIN, 18)); + + // Define a list of colors to cycle through. + List colors = + Arrays.asList( + Color.RED, + Color.GREEN, + Color.BLUE, + Color.YELLOW, + Color.CYAN, + Color.MAGENTA, + Color.ORANGE); + + for (int i = 0; i < boundingBoxes.size(); i++) { + BoundingBox boundingBox = boundingBoxes.get((i)); + List box2d = boundingBox.getBox2d(); + + // Scale normalized coordinates (0-1000) to image dimensions. + int topY = (int) (box2d.get(0) / 1000.0 * height); + int leftX = (int) (box2d.get(1) / 1000.0 * width); + int bottomY = (int) (box2d.get(2) / 1000.0 * height); + int rightX = (int) (box2d.get(3) / 1000.0 * width); + + Color color = colors.get(i % colors.size()); + graphics2D.setColor(color); + + // Draw the rectangle. + graphics2D.drawRect(leftX, topY, rightX - leftX, bottomY - topY); + + // Draw the label text. + if (boundingBox.getLabel() != null && !boundingBox.getLabel().isEmpty()) { + graphics2D.drawString(boundingBox.getLabel(), leftX + 8, topY + 20); + } + } + graphics2D.dispose(); + + // Write the image to a file. + String outputFilePath = "resources/output/bounding-boxes-socks.jpg"; + ImageIO.write(image, "jpg", new File(outputFilePath)); + System.out.println("Successfully saved image to: " + outputFilePath); + } + + public static void main(String[] args) throws IOException { + // TODO(developer): Replace these variables before running the sample. + String model = "gemini-2.5-flash"; + generateContent(model); + } + + // Shows how to send a multimodal prompt to the model and get a structured JSON response + // containing bounding box data, and then uses that data to draw the boxes on the original + // image, saving it to a new file. + public static String generateContent(String modelId) throws IOException { + // Client Initialization. Once created, it can be reused for multiple requests. + try (Client client = + Client.builder() + .location("global") + .httpOptions(HttpOptions.builder().apiVersion("v1").build()) + .vertexAI(true) + .build()) { + + String systemInstruction = + "Return bounding boxes as an array with labels.\n" + + " Never return masks. Limit to 25 objects.\n" + + " If an object is present multiple times, give each object a unique label\n" + + " according to its distinct characteristics (colors, size, position, etc..)."; + + // Define the response schema. + Schema responseSchema = + Schema.builder() + .type(ARRAY) + .items( + Schema.builder() + .type(OBJECT) + .properties( + Map.of( + "box2d", + Schema.builder() + .type(ARRAY) + .items(Schema.builder().type(INTEGER).build()) + .build(), + "label", + Schema.builder().type(STRING).build())) + .required("box2d", "label") + .build()) + .build(); + + // Define the GenerateContentConfig and set the response schema. + GenerateContentConfig contentConfig = + GenerateContentConfig.builder() + .systemInstruction(Content.fromParts(Part.fromText(systemInstruction))) + .temperature(0.5F) + .safetySettings( + SafetySetting.builder() + .category(HarmCategory.Known.HARM_CATEGORY_DANGEROUS_CONTENT) + .threshold(HarmBlockThreshold.Known.BLOCK_ONLY_HIGH) + .build()) + .responseMimeType("application/json") + .responseSchema(responseSchema) + .build(); + + String imageUri = "https://storage.googleapis.com/generativeai-downloads/images/socks.jpg"; + URL url = new URL(imageUri); + + try (InputStream inputStream = url.openStream()) { + byte[] imageBytes = inputStream.readAllBytes(); + + String prompt = + "Output the positions of the socks with a face." + + " Label according to position in the image"; + + GenerateContentResponse response = + client.models.generateContent( + modelId, + Content.fromParts(Part.fromBytes(imageBytes, "image/jpeg"), Part.fromText(prompt)), + contentConfig); + + System.out.println(response.text()); + // Example response: + // [ + // {"box2d": [24, 24, 521, 526], "label": "top left light blue cat face sock"}, + // {"box2d": [238, 627, 649, 863], "label": "top right light blue cat face sock"} + // ] + + // Use Gson to parse the JSON string into a list of BoundingBox objects. + Gson gson = new Gson(); + Type boundingBoxListType = new TypeToken>() {}.getType(); + List boundingBoxes = gson.fromJson(response.text(), boundingBoxListType); + + // Plot the bounding boxes on the image. + if (boundingBoxes != null) { + plotBoundingBoxes(imageUri, boundingBoxes); + } + + return response.text(); + } + } + } +} +// [END googlegenaisdk_boundingbox_with_txt_img] diff --git a/genai/snippets/src/main/java/genai/expressmode/ExpressModeWithApiKey.java b/genai/snippets/src/main/java/genai/expressmode/ExpressModeWithApiKey.java new file mode 100644 index 00000000000..fdb2bec0dfa --- /dev/null +++ b/genai/snippets/src/main/java/genai/expressmode/ExpressModeWithApiKey.java @@ -0,0 +1,54 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package genai.expressmode; + +// [START googlegenaisdk_vertexai_express_mode] + +import com.google.genai.Client; +import com.google.genai.types.GenerateContentConfig; +import com.google.genai.types.GenerateContentResponse; + +public class ExpressModeWithApiKey { + + public static void main(String[] args) { + // TODO(developer): Replace these variables before running the sample. + String modelId = "gemini-2.5-flash"; + String apiKey = "YOUR_API_KEY"; + generateContent(modelId, apiKey); + } + + // Generates content with Vertex AI Api key. + public static String generateContent(String modelId, String apiKey) { + // Client Initialization. Once created, it can be reused for multiple requests. + try (Client client = Client.builder().apiKey(apiKey).vertexAI(true).build()) { + + GenerateContentResponse response = + client.models.generateContent( + modelId, "Explain bubble sort to me.", GenerateContentConfig.builder().build()); + + System.out.print(response.text()); + // Example response: + // Bubble sort is one of the simplest sorting algorithms. It's often used to introduce the + // concept of sorting because its logic is very straightforward. + // + // Imagine you have a list of numbers that you want to put in order, like `[5, 1, 4, 2, 8]`. + // ... + return response.text(); + } + } +} +// [END googlegenaisdk_vertexai_express_mode] diff --git a/genai/snippets/src/test/java/genai/boundingbox/BoundingBoxIT.java b/genai/snippets/src/test/java/genai/boundingbox/BoundingBoxIT.java new file mode 100644 index 00000000000..8dc64029df3 --- /dev/null +++ b/genai/snippets/src/test/java/genai/boundingbox/BoundingBoxIT.java @@ -0,0 +1,74 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package genai.boundingbox; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class BoundingBoxIT { + + private static final String GEMINI_FLASH = "gemini-2.5-flash"; + + private ByteArrayOutputStream bout; + private PrintStream out; + + // Check if the required environment variables are set. + public static void requireEnvVar(String envVarName) { + assertWithMessage(String.format("Missing environment variable '%s' ", envVarName)) + .that(System.getenv(envVarName)) + .isNotEmpty(); + } + + @BeforeClass + public static void checkRequirements() { + requireEnvVar("GOOGLE_CLOUD_PROJECT"); + } + + @Before + public void setUp() { + bout = new ByteArrayOutputStream(); + out = new PrintStream(bout); + System.setOut(out); + } + + @After + public void tearDown() { + System.setOut(null); + } + + @Test + public void testBoundingBoxWithTxtImg() throws IOException { + String response = BoundingBoxWithTxtImg.generateContent(GEMINI_FLASH); + assertThat(response).isNotEmpty(); + String output = bout.toString(); + assertThat(output).contains("box2d"); + assertThat(output).contains("label"); + assertThat(output) + .contains("Successfully saved image to: resources/output/bounding-boxes-socks.jpg"); + } +} diff --git a/genai/snippets/src/test/java/genai/expressmode/ExpressModeIT.java b/genai/snippets/src/test/java/genai/expressmode/ExpressModeIT.java new file mode 100644 index 00000000000..2ef21de0f60 --- /dev/null +++ b/genai/snippets/src/test/java/genai/expressmode/ExpressModeIT.java @@ -0,0 +1,93 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package genai.expressmode; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.RETURNS_SELF; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.genai.Client; +import com.google.genai.Models; +import com.google.genai.types.GenerateContentConfig; +import com.google.genai.types.GenerateContentResponse; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.lang.reflect.Field; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.MockedStatic; + +@RunWith(JUnit4.class) +public class ExpressModeIT { + + private static final String GEMINI_FLASH = "gemini-2.5-flash"; + private ByteArrayOutputStream bout; + private PrintStream out; + + @Before + public void setUp() { + bout = new ByteArrayOutputStream(); + out = new PrintStream(bout); + System.setOut(out); + } + + @After + public void tearDown() { + System.setOut(null); + } + + @Test + public void testExpressModeWithApiKey() throws NoSuchFieldException, IllegalAccessException { + String response = "Bubble sort is one of the simplest sorting algorithms"; + + Client.Builder mockBuilder = mock(Client.Builder.class, RETURNS_SELF); + Client mockClient = mock(Client.class); + Models mockModels = mock(Models.class); + GenerateContentResponse mockResponse = mock(GenerateContentResponse.class); + + try (MockedStatic mockStatic = mockStatic(Client.class)) { + mockStatic.when(Client::builder).thenReturn(mockBuilder); + when(mockBuilder.build()).thenReturn(mockClient); + + // Using reflection because 'models' is a final field and cannot be mocked directly + Field field = Client.class.getDeclaredField("models"); + field.setAccessible(true); + field.set(mockClient, mockModels); + + when(mockClient.models.generateContent( + anyString(), anyString(), any(GenerateContentConfig.class))) + .thenReturn(mockResponse); + when(mockResponse.text()).thenReturn(response); + + String generatedResponse = ExpressModeWithApiKey.generateContent(GEMINI_FLASH, "API_KEY"); + + verify(mockClient.models, times(1)) + .generateContent(anyString(), anyString(), any(GenerateContentConfig.class)); + assertThat(generatedResponse).isNotEmpty(); + assertThat(response).isEqualTo(generatedResponse); + } + } +}