Expand the content of an image using outpaint

from google import genai from google.genai.types import RawReferenceImage, MaskReferenceImage, MaskReferenceConfig, EditImageConfig  client = genai.Client()  # TODO(developer): Update and un-comment below line # output_file = "output-image.png"  raw_ref = RawReferenceImage(     reference_image=Image.from_file(location='test_resources/living_room.png'), reference_id=0) mask_ref = MaskReferenceImage(     reference_id=1,     reference_image=Image.from_file(location='test_resources/living_room_mask.png'),     config=MaskReferenceConfig(         mask_mode="MASK_MODE_USER_PROVIDED",         mask_dilation=0.03,     ), )  image = client.models.edit_image(     model="imagen-3.0-capability-001",     prompt="A chandelier hanging from the ceiling",     reference_images=[raw_ref, mask_ref],     config=EditImageConfig(         edit_mode="EDIT_MODE_OUTPAINT",     ), )  image.generated_images[0].image.save(output_file)  print(f"Created output image using {len(image.generated_images[0].image.image_bytes)} bytes") # Example response: # Created output image using 1234567 bytes 

 import vertexai from vertexai.preview.vision_models import Image, ImageGenerationModel  # TODO(developer): Update and un-comment below lines # PROJECT_ID = "your-project-id" # input_file = "input-image.png" # mask_file = "mask-image.png" # output_file = "output-image.png" # prompt = "" # The optional text prompt describing what you want to see inserted.  vertexai.init(project=PROJECT_ID, location="us-central1")  model = ImageGenerationModel.from_pretrained("imagegeneration@006") base_img = Image.load_from_file(location=input_file) mask_img = Image.load_from_file(location=mask_file)  images = model.edit_image(     base_image=base_img,     mask=mask_img,     prompt=prompt,     edit_mode="outpainting", )  images[0].save(location=output_file, include_generation_parameters=False)  # Optional. View the edited image in a notebook. # images[0].show()  print(f"Created output image using {len(images[0]._image_bytes)} bytes") # Example response: # Created output image using 1234567 bytes 

 import com.google.api.gax.rpc.ApiException; import com.google.cloud.aiplatform.v1.EndpointName; import com.google.cloud.aiplatform.v1.PredictResponse; import com.google.cloud.aiplatform.v1.PredictionServiceClient; import com.google.cloud.aiplatform.v1.PredictionServiceSettings; import com.google.gson.Gson; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Value; import com.google.protobuf.util.JsonFormat; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Base64; import java.util.Collections; import java.util.HashMap; import java.util.Map;  public class EditImageOutpaintingMaskSample {    public static void main(String[] args) throws IOException {     // TODO(developer): Replace these variables before running the sample.     String projectId = "my-project-id";     String location = "us-central1";     String inputPath = "/path/to/my-input.png";     String maskPath = "/path/to/my-mask.png";     String prompt = ""; // The optional text prompt describing what you want to see inserted.      editImageOutpaintingMask(projectId, location, inputPath, maskPath, prompt);   }    // Edit an image using a mask file. Outpainting lets you expand the content of a base image to fit   // a larger or differently sized mask canvas.   public static PredictResponse editImageOutpaintingMask(       String projectId, String location, String inputPath, String maskPath, String prompt)       throws ApiException, IOException {     final String endpoint = String.format("%s-aiplatform.googleapis.com:443", location);     PredictionServiceSettings predictionServiceSettings =         PredictionServiceSettings.newBuilder().setEndpoint(endpoint).build();      // Initialize client that will be used to send requests. This client only needs to be created     // once, and can be reused for multiple requests.     try (PredictionServiceClient predictionServiceClient =         PredictionServiceClient.create(predictionServiceSettings)) {        final EndpointName endpointName =           EndpointName.ofProjectLocationPublisherModelName(               projectId, location, "google", "imagegeneration@006");        // Encode image and mask to Base64       String imageBase64 =           Base64.getEncoder().encodeToString(Files.readAllBytes(Paths.get(inputPath)));       String maskBase64 =           Base64.getEncoder().encodeToString(Files.readAllBytes(Paths.get(maskPath)));        // Create the image and image mask maps       Map<String, String> imageMap = new HashMap<>();       imageMap.put("bytesBase64Encoded", imageBase64);        Map<String, String> maskMap = new HashMap<>();       maskMap.put("bytesBase64Encoded", maskBase64);       Map<String, Map> imageMaskMap = new HashMap<>();       imageMaskMap.put("image", maskMap);        Map<String, Object> instancesMap = new HashMap<>();       instancesMap.put("prompt", prompt); // [ "prompt", "<my-prompt>" ]       instancesMap.put(           "image", imageMap); // [ "image", [ "bytesBase64Encoded", "iVBORw0KGgo...==" ] ]       instancesMap.put(           "mask",           imageMaskMap); // [ "mask", [ "image", [ "bytesBase64Encoded", "iJKDF0KGpl...==" ] ] ]       instancesMap.put("editMode", "outpainting"); // [ "editMode", "outpainting" ]       Value instances = mapToValue(instancesMap);        // Optional parameters       Map<String, Object> paramsMap = new HashMap<>();       paramsMap.put("sampleCount", 1);       Value parameters = mapToValue(paramsMap);        PredictResponse predictResponse =           predictionServiceClient.predict(               endpointName, Collections.singletonList(instances), parameters);        for (Value prediction : predictResponse.getPredictionsList()) {         Map<String, Value> fieldsMap = prediction.getStructValue().getFieldsMap();         if (fieldsMap.containsKey("bytesBase64Encoded")) {           String bytesBase64Encoded = fieldsMap.get("bytesBase64Encoded").getStringValue();           Path tmpPath = Files.createTempFile("imagen-", ".png");           Files.write(tmpPath, Base64.getDecoder().decode(bytesBase64Encoded));           System.out.format("Image file written to: %s\n", tmpPath.toUri());         }       }       return predictResponse;     }   }    private static Value mapToValue(Map<String, Object> map) throws InvalidProtocolBufferException {     Gson gson = new Gson();     String json = gson.toJson(map);     Value.Builder builder = Value.newBuilder();     JsonFormat.parser().merge(json, builder);     return builder.build();   } } 

/**  * TODO(developer): Update these variables before running the sample.  */ const projectId = process.env.CAIP_PROJECT_ID; const location = 'us-central1'; const inputFile = 'resources/roller_skaters.png'; const maskFile = 'resources/roller_skaters_mask.png'; const prompt = 'city with skyscrapers';  const aiplatform = require('@google-cloud/aiplatform');  // Imports the Google Cloud Prediction Service Client library const {PredictionServiceClient} = aiplatform.v1;  // Import the helper module for converting arbitrary protobuf.Value objects const {helpers} = aiplatform;  // Specifies the location of the api endpoint const clientOptions = {   apiEndpoint: `${location}-aiplatform.googleapis.com`, };  // Instantiates a client const predictionServiceClient = new PredictionServiceClient(clientOptions);  async function editImageOutpaintingMask() {   const fs = require('fs');   const util = require('util');   // Configure the parent resource   const endpoint = `projects/${projectId}/locations/${location}/publishers/google/models/imagegeneration@006`;    const imageFile = fs.readFileSync(inputFile);   // Convert the image data to a Buffer and base64 encode it.   const encodedImage = Buffer.from(imageFile).toString('base64');    const maskImageFile = fs.readFileSync(maskFile);   // Convert the image mask data to a Buffer and base64 encode it.   const encodedMask = Buffer.from(maskImageFile).toString('base64');    const promptObj = {     prompt: prompt, // The optional text prompt describing what you want to see inserted     editMode: 'outpainting',     image: {       bytesBase64Encoded: encodedImage,     },     mask: {       image: {         bytesBase64Encoded: encodedMask,       },     },   };   const instanceValue = helpers.toValue(promptObj);   const instances = [instanceValue];    const parameter = {     // Optional parameters     seed: 100,     // Controls the strength of the prompt     // 0-9 (low strength), 10-20 (medium strength), 21+ (high strength)     guidanceScale: 21,     sampleCount: 1,   };   const parameters = helpers.toValue(parameter);    const request = {     endpoint,     instances,     parameters,   };    // Predict request   const [response] = await predictionServiceClient.predict(request);   const predictions = response.predictions;   if (predictions.length === 0) {     console.log(       'No image was generated. Check the request parameters and prompt.'     );   } else {     let i = 1;     for (const prediction of predictions) {       const buff = Buffer.from(         prediction.structValue.fields.bytesBase64Encoded.stringValue,         'base64'       );       // Write image content to the output file       const writeFile = util.promisify(fs.writeFile);       const filename = `output${i}.png`;       await writeFile(filename, buff);       console.log(`Saved image ${filename}`);       i++;     }   } } await editImageOutpaintingMask();