产品图抠图和重绘，先提交一次

src/api/service/text2videoService.ts

@@ -73,57 +73,6 @@ export default {
       });
   },
 
-  submitSDInPaint(
-    task_id: string = "",
-    img_idx: string = "",
-    prompt: string = "",
-    negative_prompt: string = "",
-    width: string = "",
-    height: string = "",
-    sampler_index: string = "",
-    seed: string = "",
-    steps: string = "",
-    cfg_scale: string = "",
-    encrypt: string = "false",
-    model: string = "",
-    base_img: string = "",
-    mask: string = ""
-  ): Promise<{"domain_image_path": string, "local_image_path": string}> {
-    if (!prompt || !base_img || !mask) {
-      return Promise.reject("SD提示词、基础图、mask均不能为空");
-    }
-    const post_data = {
-      task_id: task_id,
-      img_idx: img_idx,
-      prompt: prompt,
-      negative_prompt: negative_prompt,
-      sampler_index: sampler_index,
-      seed: seed,
-      steps: steps,
-      width: width,
-      height: height,
-      cfg_scale: cfg_scale,
-      encrypt: encrypt,
-      model: model,
-      base_img: base_img,
-      mask: mask
-    }
-    return request.post('/text2video/img2img_inpaint', post_data)
-      .then((res: any) => {
-        // console.log(res);
-        if (res && res.code === 0) {
-          return {"domain_image_path": res.data.result.domain_image_path, "local_image_path": res.data.result.local_image_path};
-        } else {
-          const errorMessage = res ? res.message : "未知错误";
-          return Promise.reject(errorMessage);
-        }
-      })
-      .catch((err: any) => {
-        console.log(`err = ${JSON.stringify(err)}`);
-        return Promise.reject(`与 stable-diffusion-webui Api 通讯失败`);
-      });
-  },
-
   submitGenVideo(gen_video_param: any): Promise<string> {
     if (!gen_video_param) {
       return Promise.reject("输入不能为空");
@@ -258,4 +207,75 @@ export default {
         return Promise.reject(`与ImgTextMatch Api通讯失败`);
       });
   },
+
+  submitCutOutImg(param: any): Promise<any> {
+    const post_data = param;
+    return request.post('/text2video/cutout_from_img', post_data)
+      .then((res: any) => {
+        // console.log(res);
+        if (res && res.code === 0) {
+          return res.data.result;
+        } else {
+          const errorMessage = res ? res.message : "未知错误";
+          return Promise.reject(errorMessage);
+        }
+      })
+      .catch((err: any) => {
+        console.log(`err = ${JSON.stringify(err)}`);
+        return Promise.reject(`与 cutout_from_img Api 通讯失败`);
+      });
+  },
+
+  submitSDInPaint(
+    task_id: string = "",
+    img_idx: string = "",
+    prompt: string = "",
+    negative_prompt: string = "",
+    width: string = "",
+    height: string = "",
+    sampler_index: string = "",
+    seed: string = "",
+    steps: string = "",
+    cfg_scale: string = "",
+    encrypt: string = "false",
+    model: string = "",
+    base_img: string = "",
+    mask: string = "",
+    img_data_type: string = ""
+  ): Promise<{"domain_image_path": string, "local_image_path": string}> {
+    if (!prompt || !base_img || !mask) {
+      return Promise.reject("SD提示词、基础图、mask均不能为空");
+    }
+    const post_data = {
+      task_id: task_id,
+      img_idx: img_idx,
+      prompt: prompt,
+      negative_prompt: negative_prompt,
+      sampler_index: sampler_index,
+      seed: seed,
+      steps: steps,
+      width: width,
+      height: height,
+      cfg_scale: cfg_scale,
+      encrypt: encrypt,
+      model: model,
+      base_img: base_img,
+      mask: mask,
+      img_data_type: img_data_type
+    }
+    return request.post('/text2video/img2img_inpaint', post_data)
+      .then((res: any) => {
+        // console.log(res);
+        if (res && res.code === 0) {
+          return {"domain_image_path": res.data.result.domain_image_path, "local_image_path": res.data.result.local_image_path};
+        } else {
+          const errorMessage = res ? res.message : "未知错误";
+          return Promise.reject(errorMessage);
+        }
+      })
+      .catch((err: any) => {
+        console.log(`err = ${JSON.stringify(err)}`);
+        return Promise.reject(`与 img2img_inpaint Api 通讯失败`);
+      });
+  },
 }

src/views/home/index.vue

@@ -100,6 +100,14 @@ const update_and_modify_product = reactive({
   pic_local: "",
   pic_preview: "src/assets/waiting.png",
   pic_preview_local: "",
+  pic_mask: "src/assets/waiting.png",
+  pic_mask_local: "",
+  inPaintVisible: false,
+  inPaintBaseImgData: "",
+  inPaintMaskData: "",
+  inPaintPrompt: "",
+  cutout_obj: <Wm.Coordinate[]>[],
+  cutout_no_obj: <Wm.Coordinate[]>[],
 });
 
 
@@ -822,6 +830,8 @@ const clean_data = () => {
 
   update_and_modify_product.pic = "src/assets/waiting.png";
   update_and_modify_product.pic_local = "";
+  update_and_modify_product.pic_preview = "src/assets/waiting.png";
+  update_and_modify_product.pic_preview_local = "";
 
 }
 
@@ -885,6 +895,7 @@ const onChangeScreen = (val: string) => {
     // cover_backcover.cover_pic_with_text = default_data.cover_backcover.cover_pic_with_text;
     // cover_backcover.cover_pic_with_text_local = default_data.cover_backcover.cover_pic_with_text_local;
 
+    update_and_modify_product.if_need = "true";
     update_and_modify_product.pic = "http://127.0.0.1:5001/assets/2024/04/18/b7f41e18-fd2f-11ee-980d-76e065928c9e_resized.png?v=20240418112349682";
     update_and_modify_product.pic_local = "assets/2024/04/18/b7f41e18-fd2f-11ee-980d-76e065928c9e_resized.png";
   }
@@ -1470,7 +1481,6 @@ const showInPaintDialog = (type: string, item: any) => {
       console.error('Canvas element not found');
     }
   });
-
 }
 
 const onSubmitInPaint = async () => {
@@ -1540,6 +1550,7 @@ const onSubmitInPaint = async () => {
     //   model,
     //   base_img,
     //   mask,
+    //   "img_data",
     // );
     const sd_img = await text2videoService.submitSDInPaint(
       form.task_id,
@@ -1556,6 +1567,7 @@ const onSubmitInPaint = async () => {
       utils.aesEncrypt(model),
       utils.aesEncrypt(base_img),
       utils.aesEncrypt(mask),
+      "img_data",
     );
 
     if (inPaintType.value == "scene") {
@@ -1579,38 +1591,18 @@ const onSubmitInPaint = async () => {
 }
 
 
-//  上传自定义的产品图
+//  上传自定义的产品图并重绘
 const upload_update_and_modify_product = ref<UploadInstance>();
-
 const UploadAndModifyProductPicSuccess = (val: Wm.UploadResult) => {
   if (val.code == 0) {
     update_and_modify_product.pic = val.data[0].url + "?v=" + utils.genDateTimeStr();
     update_and_modify_product.pic_local = val.data[0].path;
-    // 解析内容
-    const param = {
-      task_id: form.task_id,
-      image_path: update_and_modify_product.pic_local,
-      prompt: "找到图片中的所有文字，理解图片中的商品信息和背景信息",
-    }
-    text2videoService
-    .submitImgToText(param)
-    .then((result: string) => {
-      ElMessage({
-        message: '上传成功',
-        type: 'success'
-      })
-      if(form.chatgpt_prompt == ''){
-        form.chatgpt_prompt = result + ' 根据以上信息，编写一段50字的小故事。';
-      } else {
-        form.chatgpt_prompt = result + ' 根据以上信息，' + form.chatgpt_prompt;
-      }
+    update_and_modify_product.pic_preview = val.data[0].url + "?v=" + utils.genDateTimeStr();
+    update_and_modify_product.pic_preview_local = val.data[0].path;
+    ElMessage({
+      message: '上传成功',
+      type: 'success'
     })
-    .catch((error: any) => {
-      ElMessage({
-        message: '解析失败：'+error,
-        type: "error",
-      });
-    });
   } else {
     ElMessage({
       message: '上传失败：'+val.message,
@@ -1618,6 +1610,29 @@ const UploadAndModifyProductPicSuccess = (val: Wm.UploadResult) => {
     })
   }
 }
+const onAnalyzeUploadAndModifyProductPic = () => {
+  // 解析内容
+  const param = {
+    task_id: form.task_id,
+    image_path: update_and_modify_product.pic_local,
+    prompt: "找到图片中的所有文字，理解图片中的商品信息和背景信息",
+  }
+  text2videoService
+  .submitImgToText(param)
+  .then((result: string) => {
+    if(form.chatgpt_prompt == ''){
+      form.chatgpt_prompt = result + ' 根据以上信息，编写一段50字的小故事。';
+    } else {
+      form.chatgpt_prompt = result + ' 根据以上信息，' + form.chatgpt_prompt;
+    }
+  })
+  .catch((error: any) => {
+    ElMessage({
+      message: '解析失败：'+error,
+      type: "error",
+    });
+  });
+};
 const handleUploadAndModifyProductPicExceed: UploadProps['onExceed'] = (files) => {
   // 清除已上传的文件
   upload_update_and_modify_product.value!.clearFiles()
@@ -1633,7 +1648,249 @@ const handleUploadAndModifyProductPicExceed: UploadProps['onExceed'] = (files) =
 const onClearUploadAndModifyProductPic = () => {
   update_and_modify_product.pic = 'src/assets/waiting.png';
   update_and_modify_product.pic_local = '';
+  update_and_modify_product.pic_preview = 'src/assets/waiting.png';
+  update_and_modify_product.pic_preview_local = '';
 };
+// 自定义产品图重绘Dialog
+let if_binded_events = false;
+const showUploadAndModifyProductPicInPaintDialog = () => {
+  update_and_modify_product.inPaintPrompt = "";
+  update_and_modify_product.inPaintBaseImgData = "";
+  update_and_modify_product.inPaintMaskData = "";
+  update_and_modify_product.cutout_obj = <Wm.Coordinate[]>[];
+  update_and_modify_product.cutout_no_obj = <Wm.Coordinate[]>[];
+
+  let base_img_path = update_and_modify_product.pic;
+  if (!base_img_path || base_img_path.length==0) {
+    ElMessage({
+      message: "没有基础图片，请确认",
+      type: "error",
+    });
+    return;
+  }
+  let preview_img_path = update_and_modify_product.pic_preview;
+  update_and_modify_product.inPaintVisible = true; // 打开对话框
+  // 等待元素加载完成
+  nextTick(() => {
+    // 自定义产品图局部重绘
+    const base_canvas = document.getElementById('update_and_modify_product_baseCanvas') as HTMLCanvasElement;
+    const preview_canvas = document.getElementById('update_and_modify_product_previewCanvas') as HTMLCanvasElement;
+    const mask_canvas = document.getElementById('update_and_modify_product_maskCanvas') as HTMLCanvasElement;
+    const clearButton = document.getElementById('update_and_modify_product_clearButton') as HTMLButtonElement;
+    if (base_canvas && preview_canvas && mask_canvas) {
+      // 基础图
+      let base_ctx = base_canvas.getContext('2d') as CanvasRenderingContext2D;
+      base_ctx.clearRect(0, 0, base_canvas.width, base_canvas.height);
+      var base_img = new Image();
+      base_img.crossOrigin = 'Anonymous'; // 如果图片需要用于跨域，则需要设置这个属性
+      base_img.onload = function() {
+        base_canvas.width = base_img.width;
+        base_canvas.height = base_img.height;
+        base_ctx.drawImage(base_img, 0, 0, base_canvas.width, base_canvas.height);
+      };
+      base_img.src = base_img_path;
+
+      // 预览图
+      let preview_ctx = preview_canvas.getContext('2d') as CanvasRenderingContext2D;
+      preview_ctx.clearRect(0, 0, preview_canvas.width, preview_canvas.height);
+      if (preview_img_path && preview_img_path.length != 0) {
+        var preview_img = new Image();
+        preview_img.crossOrigin = 'Anonymous'; // 如果图片需要用于跨域，则需要设置这个属性
+        preview_img.onload = function() {
+          preview_canvas.width = preview_img.width;
+          preview_canvas.height = preview_img.height;
+          preview_ctx.drawImage(preview_img, 0, 0, preview_canvas.width, preview_canvas.height);
+        };
+        preview_img.src = preview_img_path;
+      }
+
+      // 产品层mask
+      let mask_ctx = mask_canvas.getContext('2d') as CanvasRenderingContext2D;
+      mask_ctx.clearRect(0, 0, mask_canvas.width, mask_canvas.height);
+      mask_ctx.lineWidth = 40; // 设置线条粗细
+      mask_ctx.strokeStyle = 'rgba(255, 255, 255, 1)'; // 设置线条颜色
+      mask_ctx.lineCap = 'round';  // 线头尾为圆形
+      mask_ctx.lineJoin = 'round'; // 拐点为圆形，默认是尖角
+      mask_ctx.globalCompositeOperation = 'copy';
+
+      if (!if_binded_events) {
+        // 鼠标或触摸事件开始绘制
+        mask_canvas.addEventListener('mousedown', startDrawing);
+        // 清除按钮点击事件
+        clearButton.addEventListener('click', clearCanvas);
+        // 每次打开都会再绑定一次，这里防止多次绑定，否则数据会重复
+        if_binded_events = true;
+      }
+
+      // 开始绘制
+      function startDrawing(e: any) {
+          // isDrawing = true;
+          mask_ctx.beginPath();
+          const { offsetX, offsetY } = getOffset(e);
+          mask_ctx.moveTo(offsetX, offsetY);
+          mask_ctx.lineTo(offsetX, offsetY);
+          mask_ctx.stroke();
+          console.log(offsetX, offsetY)
+          update_and_modify_product.cutout_obj.push({x: offsetX, y: offsetY})
+      }
+      // 清除画布
+      function clearCanvas() {
+        mask_ctx.clearRect(0, 0, mask_canvas.width, mask_canvas.height);
+      }
+      // 获取鼠标的偏移量
+      function getOffset(e: any) {
+          // console.log(e)
+          let offsetX = e.offsetX;
+          let offsetY = e.offsetY;
+          // console.log({ offsetX, offsetY })
+          return { offsetX, offsetY };
+      }
+    } else {
+      ElMessage({
+        message: 'Canvas element not found',
+        type: "error",
+      });
+      console.error('Canvas element not found');
+    }
+  });
+}
+
+const onSubmitUploadAndModifyProductPicCutOut = async () => {
+  if (!form.task_id) {
+    ElMessage({
+      message: "task_id不能为空，请刷新页面",
+      type: "error",
+    });
+    return;
+  }
+  if (update_and_modify_product.cutout_obj.length==0) {
+    ElMessage({
+      message: "没有产品坐标，请在图片上点击产品！",
+      type: "error",
+    });
+    return;
+  }
+  let point_prompt: any[] = [];
+  let point_label: any[] = [];
+  // 产品坐标
+  // console.log(update_and_modify_product.cutout_obj)
+  update_and_modify_product.cutout_obj.forEach((item, index) => {
+    point_prompt.push([item.x, item.y]);
+    point_label.push(1);
+  });
+  // 非产品坐标
+  update_and_modify_product.cutout_no_obj.forEach((item, index) => {
+    point_prompt.push([item.x, item.y]);
+    point_label.push(0);
+  });
+  // console.log(point_prompt, point_label)
+  // console.log(JSON.stringify(point_prompt), JSON.stringify(point_label))
+  // return;
+  const param = {
+    task_id: form.task_id,
+    image_path: update_and_modify_product.pic_local,
+    point_prompt: JSON.stringify(point_prompt),
+    point_label: JSON.stringify(point_label),
+  }
+  // console.log(param)
+  try {
+    const cutout_result = await text2videoService.submitCutOutImg(param);
+    update_and_modify_product.pic_preview = cutout_result.domain_image_path_preview + "?v=" + utils.genDateTimeStr();
+    update_and_modify_product.pic_preview_local = cutout_result.local_image_path_preview;
+    update_and_modify_product.pic_mask = cutout_result.domain_image_path_mask + "?v=" + utils.genDateTimeStr();
+    update_and_modify_product.pic_mask_local = cutout_result.local_image_path_mask;
+  } catch (error) {
+    ElMessage({
+      message: String(error),
+      type: "error",
+    });
+  } finally {
+    update_and_modify_product.inPaintVisible = false; // 关闭对话框
+    showUploadAndModifyProductPicInPaintDialog();  // 再打开对话框进行预览
+    // update_and_modify_product.inPaintVisible = true;
+  }
+}
+
+const onSubmitUploadAndModifyProductPicInPaint = async () => {
+  if (!form.task_id) {
+    ElMessage({
+      message: "task_id不能为空，请刷新页面",
+      type: "error",
+    });
+    return;
+  }
+  if (!update_and_modify_product.pic_local || !update_and_modify_product.pic_mask_local) {
+    ElMessage({
+      message: "缺少基本图片和mask图片！",
+      type: "error",
+    });
+    return;
+  }
+  if (!update_and_modify_product.inPaintPrompt || update_and_modify_product.inPaintPrompt.length==0) {
+    ElMessage({
+      message: "请填写画面描述！",
+      type: "error",
+    });
+    return;
+  }
+  const sampler_index = sd_model.sampler_index;
+  const seed = sd_model.seed;
+  const steps = sd_model.steps;
+  const cfg_scale = sd_model.cfg_scale;
+  const model = sd_model.model;
+  const base_img = update_and_modify_product.pic_local;
+  const mask = update_and_modify_product.pic_mask_local;
+  let img_id = "update_and_modify_product"
+  update_and_modify_product.pic = 'src/assets/waiting.png';
+  update_and_modify_product.pic_local = '';
+  try {
+    let keywords_en = await text2videoService.submitTranslateToEn(utils.aesEncrypt(update_and_modify_product.inPaintPrompt), form.task_id, "true");
+    let sd_prompt = keywords_en.replace(/"/g, '') + "," + sd_prompt_prefix;
+    const sd_img = await text2videoService.submitSDInPaint(
+      form.task_id,
+      img_id,
+      sd_prompt,
+      sd_negative_prompt_prefix,
+      form.img_size.width,
+      form.img_size.height,
+      sampler_index,
+      seed,
+      steps,
+      cfg_scale,
+      "false",
+      model,
+      base_img,
+      mask,
+      "img_path",
+    );
+    // const sd_img = await text2videoService.submitSDInPaint(
+    //   form.task_id,
+    //   img_id,
+    //   utils.aesEncrypt(sd_prompt),
+    //   utils.aesEncrypt(sd_negative_prompt_prefix),
+    //   utils.aesEncrypt(form.img_size.width),
+    //   utils.aesEncrypt(form.img_size.height),
+    //   utils.aesEncrypt(sampler_index),
+    //   utils.aesEncrypt(seed),
+    //   utils.aesEncrypt(steps),
+    //   utils.aesEncrypt(cfg_scale),
+    //   "true",
+    //   utils.aesEncrypt(model),
+    //   utils.aesEncrypt(base_img),
+    //   utils.aesEncrypt(mask),
+    //   "img_path",
+    // );
+    update_and_modify_product.pic = sd_img.domain_image_path + "?v=" + utils.genDateTimeStr();
+    update_and_modify_product.pic_local = sd_img.local_image_path;
+  } catch (error) {
+    ElMessage({
+      message: String(error),
+      type: "error",
+    });
+  } finally {
+    update_and_modify_product.inPaintVisible = false; // 关闭对话框
+  }
+}
 </script>
 
 <template>
@@ -1690,8 +1947,15 @@ const onClearUploadAndModifyProductPic = () => {
             :data="{ width: form.img_size.width, height: form.img_size.height }"
             :before-upload="handleBeforeUpload"
           >
-            <el-button type="primary" size="small">上传图片并解析内容</el-button>
+            <el-button type="primary" size="small">上传图片</el-button>
           </el-upload>
+          <el-button
+            type="primary"
+            size="small"
+            @click="onAnalyzeUploadAndModifyProductPic"
+            style="margin-left: 12px"
+            >解析图片</el-button
+          >
           <el-button
             plain
             size="small"
@@ -1699,13 +1963,13 @@ const onClearUploadAndModifyProductPic = () => {
             style="margin-left: 12px"
             >清除图片</el-button
           >
-          <!-- <el-button
+          <el-button
             type="primary"
             size="small"
-            @click="showInPaintDialog('product', '')"
+            @click="showUploadAndModifyProductPicInPaintDialog"
             style="margin-left: 12px"
             >局部重绘</el-button
-          > -->
+          >
         </el-form-item>
       </div>
       <!-- Prompt到文案 -->
@@ -2497,6 +2761,72 @@ const onClearUploadAndModifyProductPic = () => {
         </div>
       </template>
     </el-dialog>
+
+    <!-- 自定义产品图局部重绘 -->
+    <el-dialog
+      v-model="update_and_modify_product.inPaintVisible"
+      :width="parseInt(form.img_size.width) + 40"
+      :close-on-click-modal="false"
+      :close-on-press-escape="true"
+      :lock-scroll="true"
+    >
+      <div style="color: red">请在图片上点击产品部分</div>
+      <div
+        ref="update_and_modify_product_inpaint"
+        :style="
+          'position: relative; width: ' +
+          form.img_size.width +
+          'px; height: ' +
+          form.img_size.height +
+          'px; margin: 10px auto;'
+        "
+      >
+        <canvas
+          id="update_and_modify_product_baseCanvas"
+          :width="form.img_size.width"
+          :height="form.img_size.height"
+          style="position: absolute; left: 0; top: 0; z-index: 1; background: none"
+        ></canvas>
+        <canvas
+          id="update_and_modify_product_previewCanvas"
+          :width="form.img_size.width"
+          :height="form.img_size.height"
+          style="position: absolute; left: 0; top: 0; z-index: 2; background: none"
+        ></canvas>
+        <canvas
+          id="update_and_modify_product_maskCanvas"
+          :width="form.img_size.width"
+          :height="form.img_size.height"
+          style="position: absolute; left: 0; top: 0; z-index: 3; background: none"
+        ></canvas>
+        <!-- <canvas
+          id="update_and_modify_product_mask2Canvas"
+          :width="form.img_size.width"
+          :height="form.img_size.height"
+          style="position: absolute; left: 0; top: 0; z-index: 4; background: none"
+        ></canvas> -->
+      </div>
+      <button id="update_and_modify_product_clearButton">清除坐标</button>
+      <el-button @click="onSubmitUploadAndModifyProductPicCutOut">智能抠图</el-button>
+      <div>
+        <span>画面描述：</span>
+        <el-input
+          v-model="update_and_modify_product.inPaintPrompt"
+          :autosize="true"
+          type="textarea"
+        ></el-input>
+      </div>
+      <template #footer>
+        <div class="dialog-footer">
+          <el-button @click="update_and_modify_product.inPaintVisible = false"
+            >取消</el-button
+          >
+          <el-button type="primary" @click="onSubmitUploadAndModifyProductPicInPaint"
+            >重绘</el-button
+          >
+        </div>
+      </template>
+    </el-dialog>
   </main>
 </template>
 

typings/types/wm/lib.wm.api.d.ts

@@ -63,4 +63,9 @@ declare namespace Wm {
     ],
     "message": string
   }
+
+  interface Coordinate {
+    "x": int,
+    "y": int,
+  }
 }