feat(v1.0.2): 增加识别记录上传

common/src/main/java/com/wmdigit/common/model/IdentifyRecordDTO.java

+package com.wmdigit.common.model;
+
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ * 识别结果DTO
+ * @author dizi
+ */
+public class IdentifyRecordDTO implements Serializable {
+
+    private int[][] rectArray;
+
+    private String[] productNames;
+
+    private String[] productCodes;
+
+    public IdentifyRecordDTO() {
+    }
+
+    public IdentifyRecordDTO(int[][] rectArray, String[] productNames, String[] productCodes) {
+        this.rectArray = rectArray;
+        this.productNames = productNames;
+        this.productCodes = productCodes;
+    }
+
+    public int[][] getRectArray() {
+        return rectArray;
+    }
+
+    public void setRectArray(int[][] rectArray) {
+        this.rectArray = rectArray;
+    }
+
+    public String[] getProductNames() {
+        return productNames;
+    }
+
+    public void setProductNames(String[] productNames) {
+        this.productNames = productNames;
+    }
+
+    public String[] getProductCodes() {
+        return productCodes;
+    }
+
+    public void setProductCodes(String[] productCodes) {
+        this.productCodes = productCodes;
+    }
+}

common/src/main/java/com/wmdigit/common/utils/BitmapUtils.java

@@ -149,6 +149,7 @@ public class BitmapUtils {
     public static void saveBitmap(Bitmap bitmap, String path) {
         try {
             File file = new File(path);
+            file.getParentFile().mkdirs();
             FileOutputStream fos = new FileOutputStream(file);
             bitmap.compress(Bitmap.CompressFormat.JPEG, 100, fos);
             fos.flush();

common/src/main/java/com/wmdigit/common/view/imageview/CropView.java

@@ -1165,6 +1165,9 @@ public class CropView extends AppCompatImageView {
         if (presetCropCoordinate == null){
             return;
         }
+        if (mImageRectF == null){
+            return;
+        }
         int l = presetCropCoordinate.left;
         int t = presetCropCoordinate.top;
         int w = presetCropCoordinate.right  - l;

core/src/main/cpp/CMakeLists.txt

@@ -56,7 +56,7 @@ set_target_properties(
         ${CMAKE_SOURCE_DIR}/../jniLibs/${ANDROID_ABI}/libret.so
 )
 
-# 分类、特征提取库
+# 菜品分类、特征提取库
 add_library(clsretri SHARED IMPORTED)
 set_target_properties(
         clsretri

core/src/main/cpp/catering_dish_detection.cpp

@@ -28,7 +28,7 @@ Java_com_wmdigit_core_catering_dish_DishDetection_process(JNIEnv *env, jobject t
     LOGD("准备推理菜品");
     // 推理
     int ret = DETFEA_Process(input, &output, handle);
-    LOGD("推理结果：%d", ret);
+    LOGD("推理结果：%x", ret);
     if (ret != 0){
         mat.release();
         mat_bgr.release();
@@ -40,7 +40,7 @@ Java_com_wmdigit_core_catering_dish_DishDetection_process(JNIEnv *env, jobject t
     int cols = 4;
     // 统计符合阈值的框数量
     for (size_t i = 0; i < output.output_list.size(); ++i) {
-        if (output.output_list[i].prob >= 0.7){
+        if (output.output_list[i].prob >= 0.8){
             rows ++;
         }
     }
@@ -59,7 +59,7 @@ Java_com_wmdigit_core_catering_dish_DishDetection_process(JNIEnv *env, jobject t
     // 遍历识别结果
     for (size_t i = 0; i < output.output_list.size(); ++i) {
         // 根据阈值，筛选出有效的框
-        if (output.output_list[i].prob >= 0.75){
+        if (output.output_list[i].prob >= 0.8){
             // 记录框的左上、右下点坐标
             rect_array[i][0] = (int)output.output_list[i].x1;
             rect_array[i][1] = (int)output.output_list[i].y1;

core/src/main/cpp/features_hnsw.cpp

@@ -5,9 +5,10 @@
  */
 extern "C"
 JNIEXPORT jint JNICALL
-Java_com_wmdigit_core_hnsw_Hnsw_init(JNIEnv *env, jobject thiz) {
+Java_com_wmdigit_core_hnsw_Hnsw_init(JNIEnv *env, jobject thiz, jint dimension) {
     std::vector<std::pair<long, std::string>>().swap(vector);
-    idx = new libhnsw::Index(FEATURES_DIMENSION, FEATURES_TOTAL, mode);
+    idx = new libhnsw::Index(dimension, FEATURES_TOTAL, mode);
+    output_dimension = dimension;
     LOGI("索引库初始化完成");
     return 0;
 }
@@ -96,7 +97,7 @@ Java_com_wmdigit_core_hnsw_Hnsw_retrieveMostSimilarResult(JNIEnv *env, jobject t
         libhnsw::RequireSample requireSample;
         idx->Selectsample(requireSample, label);
         // 计算相似度
-        float similarity = calculateSimilar(array, requireSample.data, FEATURES_DIMENSION);
+        float similarity = calculateSimilar(array, requireSample.data, output_dimension);
         if (similarity >= threshold){
             // 相似度大于设定阈值，在vector中检索出对应的code
             for (auto it = vector.begin(); it != vector.end(); it++) {

core/src/main/cpp/include/features_hnsw.h

@@ -16,8 +16,6 @@
 #define LOGE(...)  __android_log_print(ANDROID_LOG_ERROR,LOG_TAG, __VA_ARGS__)
 #define LOGF(...)  __android_log_print(ANDROID_LOG_FATAL,LOG_TAG, __VA_ARGS__)
 
-// 特征维度
-#define FEATURES_DIMENSION 160
 // 特征总数
 #define FEATURES_TOTAL 50000
 // 索引
@@ -33,6 +31,8 @@ libhnsw::SPACE_TYPE st = libhnsw::L2_DISTANCE;
 // second->商品code
 std::vector<std::pair<long, std::string>> vector;
 
+int output_dimension = 128;
+
 /**
  * 计算相似度
  * @param v1

core/src/main/java/com/wmdigit/core/CoreModule.java

@@ -36,7 +36,7 @@ public class CoreModule {
         // 初始化视频流
         VideoPipeRepository.getInstance().initVideoPipe();
         // 初始化索引库
-        HnswRepository.getInstance().init();
+        HnswRepository.getInstance().init(TargetDetectionRepository.getInstance().getDimension());
     }
 
     public static Context getAppContext() {

core/src/main/java/com/wmdigit/core/catering/TargetDetection.java

@@ -33,4 +33,9 @@ public interface TargetDetection {
      */
     int getAiMode();
 
+    /**
+     * 获取输出维度
+     * @return
+     */
+    int getDimension();
 }

core/src/main/java/com/wmdigit/core/catering/TargetDetectionRepository.java

@@ -95,4 +95,12 @@ public class TargetDetectionRepository {
     public void close(){
         targetDetection.close();
     }
+
+    /**
+     * 获取向量维度
+     * @return
+     */
+    public int getDimension(){
+        return targetDetection.getDimension();
+    }
 }

core/src/main/java/com/wmdigit/core/catering/dish/DishDetection.java

@@ -16,6 +16,11 @@ public class DishDetection implements TargetDetection {
         System.loadLibrary("catering_dish_detection");
     }
 
+    /**
+     * 输出维度
+     */
+    private final int OUTPUT_DIMENSION = 160;
+
     /**
      * JNI初始化目标检测算法
      * @return
@@ -76,4 +81,9 @@ public class DishDetection implements TargetDetection {
         return TargetDetectionRepository.AI_MODE_DISH_DETECTION;
     }
 
+    @Override
+    public int getDimension() {
+        return OUTPUT_DIMENSION;
+    }
+
 }

core/src/main/java/com/wmdigit/core/catering/plate/PlateDetection.java

@@ -16,6 +16,11 @@ public class PlateDetection implements TargetDetection {
         System.loadLibrary("catering_plate_detection");
     }
 
+    /**
+     * 算法输出维度
+     */
+    private final int OUTPUT_DIMENSION = 160;
+
     /**
      * JNI初始化目标检测算法
      * @return
@@ -80,4 +85,9 @@ public class PlateDetection implements TargetDetection {
     public int getAiMode() {
         return TargetDetectionRepository.AI_MODE_PLATE_DETECTION;
     }
+
+    @Override
+    public int getDimension() {
+        return OUTPUT_DIMENSION;
+    }
 }

core/src/main/java/com/wmdigit/core/hnsw/Hnsw.java

@@ -14,9 +14,10 @@ public class Hnsw {
 
     /**
      * 初始化方法
+     * @param dimension
      * @return
      */
-    private native int init();
+    private native int init(int dimension);
 
     /**
      * 写入索引
@@ -45,9 +46,9 @@ public class Hnsw {
      */
     private final Object syncLock = new Object();
 
-    public void initHnsw(){
+    public void initHnsw(int dimension){
         synchronized (syncLock){
-            int ret = init();
+            int ret = init(dimension);
             XLog.i("hnsw初始化结果：" + ret);
         }
     }

core/src/main/java/com/wmdigit/core/hnsw/HnswRepository.java

@@ -3,6 +3,7 @@ package com.wmdigit.core.hnsw;
 import android.text.TextUtils;
 
 import com.elvishew.xlog.XLog;
+import com.wmdigit.core.catering.TargetDetectionRepository;
 import com.wmdigit.data.database.entity.FeaturesPO;
 import com.wmdigit.data.database.entity.ProductsPO;
 import com.wmdigit.data.database.repository.FeaturesRepository;
@@ -44,6 +45,8 @@ public class HnswRepository {
     private boolean initComplete = false;
     private OnHnswInitListener listener;
 
+    private int dimension = 128;
+
     public HnswRepository() {
         hnsw = new Hnsw();
         compositeDisposable = new CompositeDisposable();
@@ -52,9 +55,10 @@ public class HnswRepository {
     /**
      * 初始化
      */
-    public void init(){
+    public void init(int dimension){
+        this.dimension = dimension;
         initComplete = false;
-        hnsw.initHnsw();
+        hnsw.initHnsw(dimension);
         Disposable disposable = Observable.create(emitter -> {
                     // 遍历特征库
                     queryAndWriteFeaturesIntoHnsw();
@@ -125,6 +129,7 @@ public class HnswRepository {
 
     /**
      * 查询并写入特征
+     * @param dimension
      */
     private void queryAndWriteFeaturesIntoHnsw(){
         int page = 1;
@@ -133,10 +138,10 @@ public class HnswRepository {
             List<FeaturesPO> list = FeaturesRepository.getInstance().getFeaturesByPage(page, pageSize);
             for (FeaturesPO featuresPO : list){
                 String[] featureStrArray = featuresPO.getFeature().split(",");
-                if (featureStrArray.length != 160){
+                if (featureStrArray.length != dimension){
                     continue;
                 }
-                float[] feature = new float[160];
+                float[] feature = new float[dimension];
                 for (int i = 0; i < feature.length; i++){
                     feature[i] = Float.parseFloat(featureStrArray[i].trim().replace("[", "").replace("]", ""));
                 }
@@ -158,7 +163,7 @@ public class HnswRepository {
         }
         close();
         hnsw.deleteAll();
-        hnsw.initHnsw();
+        hnsw.initHnsw(dimension);
     }
 
     public void setListener(OnHnswInitListener listener) {

data-local/src/main/java/com/wmdigit/data/disk/repository/DiskRepository.java

+package com.wmdigit.data.disk.repository;
+
+import android.content.Context;
+import android.graphics.Bitmap;
+import android.text.TextUtils;
+
+import com.wmdigit.common.utils.BitmapUtils;
+import com.wmdigit.data.LocalDataModule;
+import com.wmdigit.data.database.entity.ProductsPO;
+
+import java.io.File;
+import java.util.List;
+import java.util.UUID;
+
+public class DiskRepository {
+
+    private static DiskRepository instance;
+    public static DiskRepository getInstance(){
+        if (instance == null){
+            synchronized (DiskRepository.class){
+                if (instance == null){
+                    instance = new DiskRepository();
+                }
+            }
+        }
+        return instance;
+    }
+
+    private DiskRepository(){}
+
+    /**
+     * 保存识别记录到磁盘
+     * @param bitmap
+     * @param rectArray
+     * @param products
+     * @return 返回原图的路径
+     */
+    public String saveIdentifyRecordToDisk(Bitmap bitmap, int[][] rectArray, List<ProductsPO> products){
+        Context context = LocalDataModule.getAppContext();
+        // 识别记录根目录
+        String identifyRecordsRootPath = context.getExternalFilesDir("IdentifyRecords").getAbsolutePath();
+        // 商品的学习记录根目录
+        String productLearningRecordsRootPath = context.getExternalFilesDir("ProductLearningRecords").getAbsolutePath();
+        String imageFilename = UUID.randomUUID().toString() + ".jpg";
+        // 保存图片
+        BitmapUtils.saveBitmap(bitmap, identifyRecordsRootPath + "/" + imageFilename);
+        // 根据框的坐标，拆分原图
+        for (int i = 0; i < rectArray.length; i++) {
+            if (products.get(i) == null || TextUtils.isEmpty(products.get(i).getProductCode())){
+                continue;
+            }
+            Bitmap bitmapTemp = Bitmap.createBitmap(bitmap, rectArray[i][0], rectArray[i][1], rectArray[i][2] - rectArray[i][0], rectArray[i][3] - rectArray[i][1]);
+            // 保存裁出来的图片
+            BitmapUtils.saveBitmap(bitmapTemp, productLearningRecordsRootPath + "/" + products.get(i).getProductCode() + "/" + imageFilename.replace(".jpg", "") + "_" + i + ".jpg");
+            bitmapTemp.recycle();
+        }
+
+        return identifyRecordsRootPath + "/" + imageFilename;
+    }
+
+}

data-local/src/main/java/com/wmdigit/data/mmkv/repository/UserLocalRepository.java

@@ -107,4 +107,12 @@ public class UserLocalRepository {
         return result;
     }
 
+    /**
+     * 获取PosId
+     * @return
+     */
+    public String getPosId(){
+        return MMKV.defaultMMKV().getString(MmkvCons.MMKV_KEY_DEVICE_ID, "");
+    }
+
 }

data-remote/src/main/java/com/wmdigit/network/bean/request/CateringIdentifyRecord.java

+package com.wmdigit.network.bean.request;
+
+import java.io.Serializable;
+
+/**
+ * 餐饮识别记录
+ * @author dizi
+ */
+public class CateringIdentifyRecord implements Serializable {
+
+    /**
+     * POS机ID，该值为后台PosMachine的主键
+     */
+    private String posCode;
+
+    /**
+     * 框图URL
+     */
+    private String frameImageUrl;
+
+    /**
+     * 原图URL
+     */
+    private String imageUrl;
+
+    /**
+     * 裁剪坐标JSON
+     */
+    private String smallImageSite;
+
+    /**
+     * 识别时间
+     * yyyy-MM-dd HH:mm:ss
+     */
+    private String identifyTime;
+
+    public CateringIdentifyRecord() {
+    }
+
+    public CateringIdentifyRecord(String posCode, String frameImageUrl, String imageUrl, String smallImageSite, String identifyTime) {
+        this.posCode = posCode;
+        this.frameImageUrl = frameImageUrl;
+        this.imageUrl = imageUrl;
+        this.smallImageSite = smallImageSite;
+        this.identifyTime = identifyTime;
+    }
+
+    public String getPosCode() {
+        return posCode;
+    }
+
+    public void setPosCode(String posCode) {
+        this.posCode = posCode;
+    }
+
+    public String getFrameImageUrl() {
+        return frameImageUrl;
+    }
+
+    public void setFrameImageUrl(String frameImageUrl) {
+        this.frameImageUrl = frameImageUrl;
+    }
+
+    public String getImageUrl() {
+        return imageUrl;
+    }
+
+    public void setImageUrl(String imageUrl) {
+        this.imageUrl = imageUrl;
+    }
+
+    public String getSmallImageSite() {
+        return smallImageSite;
+    }
+
+    public void setSmallImageSite(String smallImageSite) {
+        this.smallImageSite = smallImageSite;
+    }
+
+    public String getIdentifyTime() {
+        return identifyTime;
+    }
+
+    public void setIdentifyTime(String identifyTime) {
+        this.identifyTime = identifyTime;
+    }
+}

data-remote/src/main/java/com/wmdigit/network/repository/IdentifyRecordRepository.java

+package com.wmdigit.network.repository;
+
+import com.elvishew.xlog.XLog;
+import com.wmdigit.common.utils.DateUtils;
+import com.wmdigit.data.mmkv.repository.UserLocalRepository;
+import com.wmdigit.network.bean.request.CateringIdentifyRecord;
+import com.wmdigit.network.bean.response.BasePosResponse;
+import com.wmdigit.network.factory.ServiceFactory;
+import com.wmdigit.network.oss.OSSManager;
+import com.wmdigit.network.utils.RxHelper;
+
+import java.io.File;
+
+import io.reactivex.Observable;
+import io.reactivex.Observer;
+import io.reactivex.disposables.Disposable;
+import io.reactivex.functions.Consumer;
+
+public class IdentifyRecordRepository {
+
+    private static IdentifyRecordRepository instance;
+
+    public static IdentifyRecordRepository getInstance(){
+        if (instance == null){
+            synchronized (IdentifyRecordRepository.class){
+                if (instance == null){
+                    instance = new IdentifyRecordRepository();
+                }
+            }
+        }
+        return instance;
+    }
+
+    private IdentifyRecordRepository(){
+    }
+
+    /**
+     * 上传识别记录
+     * 此方法用于上传用户的识别记录到服务器，包括图片的同步上传和识别记录的异步处理
+     *
+     * @param json 识别记录的JSON字符串，包含识别相关信息
+     * @param imagePath 待上传图片的路径
+     * @return 返回一个Disposable对象，用于取消订阅
+     */
+    public Disposable uploadIdentifyRecord(String json, String imagePath){
+        return Observable.create(emitter -> {
+            // 同步上传图片到对象存储服务
+            String url = OSSManager.getInstance().syncUploadFileToOSS(new File(imagePath));
+            // 向观察者发射上传后的图片URL
+            emitter.onNext(url);
+        }).flatMap(url ->{
+            // 创建一个餐饮识别记录对象
+            CateringIdentifyRecord record = new CateringIdentifyRecord();
+            // 设置POS机编号
+            record.setPosCode(UserLocalRepository.getInstance().getPosId());
+            // 设置识别时间
+            record.setIdentifyTime(DateUtils.getTodayTime());
+            // 设置图片URL
+            record.setImageUrl((String) url);
+            // 设置帧图片URL
+            record.setFrameImageUrl((String) url);
+            // 设置小图片站点信息
+            record.setSmallImageSite(json);
+            // 异步上传餐饮识别记录到服务器
+            return ServiceFactory.getServiceFactory().getPosService()
+                    .uploadCateringIdentifyRecord(record)
+                    .compose(RxHelper.handlePosResult());
+        }).subscribe(aLong -> {
+            // 日志记录上传成功
+            XLog.i("上传成功");
+        }, throwable -> {
+            XLog.e(throwable.toString());
+        });
+    }
+
+}

data-remote/src/main/java/com/wmdigit/network/service/PosService.java

 package com.wmdigit.network.service;
 
+import com.wmdigit.network.bean.request.CateringIdentifyRecord;
 import com.wmdigit.network.bean.request.ProductIdentifyRecord;
 import com.wmdigit.network.bean.request.QueryCommandParam;
 import com.wmdigit.network.bean.request.QueryLatestAppVersionParam;
@@ -94,4 +95,12 @@ public interface PosService {
     @GET("newretail/api/sys/app/version/getLatestWithType")
     Observable<BasePosResponse<AppVersionDTO>> getLatestRemoteToolWithType(@Query("type") String type);
 
+    /**
+     * 上传餐饮识别记录
+     * @param record
+     * @return
+     */
+    @POST("newretail/api/search/cateringIdentifyRecord/saveCateringIdentifyRecord")
+    Observable<BasePosResponse<Long>> uploadCateringIdentifyRecord(@Body CateringIdentifyRecord record);
+
 }

history.txt

@@ -10,3 +10,5 @@ v1.0.2  2024/08/06  1.增加系统信息页
                     7.增加AIDL服务
                     8.集成标框、菜品识别、餐盘识别算法
                     9.集成索引库算法（索引库版本较老，可能存在最后一条索引删除不掉的BUG）
+                    10.todo 菜品算法更新128模型
+                    11.增加数据上传
\ No newline at end of file

module-setting/src/main/java/com/wmdigit/setting/viewmodel/DataLearningViewModel.java

@@ -8,10 +8,12 @@ import androidx.annotation.NonNull;
 import androidx.databinding.ObservableField;
 import androidx.lifecycle.MutableLiveData;
 
+import com.google.gson.Gson;
 import com.wmdigit.camera.listener.OnImageAnalyzeListener;
 import com.wmdigit.common.base.mvvm.BaseViewModel;
 import com.wmdigit.common.base.mvvm.SingleLiveEvent;
 import com.wmdigit.common.model.CropValueDTO;
+import com.wmdigit.common.model.IdentifyRecordDTO;
 import com.wmdigit.common.model.ProductsVO;
 import com.wmdigit.common.utils.ParcelHelper;
 import com.wmdigit.core.catering.TargetDetectionRepository;
@@ -23,7 +25,10 @@ import com.wmdigit.data.database.entity.ProductsPO;
 import com.wmdigit.data.database.mapper.ProductsMapper;
 import com.wmdigit.data.database.repository.FeaturesRepository;
 import com.wmdigit.data.database.repository.ProductsRepository;
+import com.wmdigit.data.disk.repository.DiskRepository;
 import com.wmdigit.data.mmkv.repository.CropLocalRepository;
+import com.wmdigit.network.oss.OSSManager;
+import com.wmdigit.network.repository.IdentifyRecordRepository;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -32,6 +37,10 @@ import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 
+import io.reactivex.Observable;
+import io.reactivex.ObservableEmitter;
+import io.reactivex.ObservableOnSubscribe;
+
 /**
  * 学习菜品的ViewModel
  * @author dizi
@@ -395,7 +404,11 @@ public class DataLearningViewModel extends BaseViewModel {
             if (detectResult == null || detectResult.getProducts() == null){
                 return;
             }
+            String[] productNames = new String[detectResult.getProducts().size()];
+            String[] productCodes = new String[detectResult.getProducts().size()];
             for (int i = 0; i < detectResult.getProducts().size(); i++){
+                productNames[i] = "";
+                productCodes[i] = "";
                 if (TextUtils.isEmpty(detectResult.getProducts().get(i).getProductCode())){
                     continue;
                 }
@@ -413,11 +426,19 @@ public class DataLearningViewModel extends BaseViewModel {
                 if (id != -1) {
                     HnswRepository.getInstance().writeFeatureIntoHnsw(id, detectResult.getProducts().get(i).getProductCode(), detectResult.getFeatures()[i]);
                 }
+                productNames[i] = detectResult.getProducts().get(i).getProductName();
+                productCodes[i] = detectResult.getProducts().get(i).getProductCode();
             }
+            String imagePath = DiskRepository.getInstance().saveIdentifyRecordToDisk(detectResult.getBitmap(), detectResult.getRectArray(), detectResult.getProducts());
             toastMessage.postValue(getApplication().getString(com.wmdigit.common.R.string.save_success));
+            // 异步上传后台
+            IdentifyRecordDTO identifyRecordDTO = new IdentifyRecordDTO(detectResult.getRectArray(), productNames, productCodes);
+            String json = new Gson().toJson(identifyRecordDTO);
+            compositeDisposable.add(IdentifyRecordRepository.getInstance().uploadIdentifyRecord(json, imagePath));
         }
     }
 
+
     public OnImageAnalyzeListener getOnImageAnalyzeListener() {
         return onImageAnalyzeListener;
     }