Skip to content

A
Android_Catering_service
Commit 719611fb authored by 姜天宇's avatar 姜天宇
Browse files
Options

feat(v1.0.1): 增加3568视频流算法

parent ad40cf69
Hide whitespace changes
Inline Side-by-side
Showing with 1590 additions and 10 deletions
.gitignore
View file @ 719611fb
...@@ -8,8 +8,15 @@ ...@@ -8,8 +8,15 @@
/.idea/navEditor.xml /.idea/navEditor.xml
/.idea/assetWizardSettings.xml /.idea/assetWizardSettings.xml
.DS_Store .DS_Store
/build
/captures /captures
.externalNativeBuild .externalNativeBuild
.cxx .cxx
local.properties local.properties
/build
/core/build/
/common/build/
/camera/build/
/data-local/build/
/data-remote/build/
/module-demo/build/
/.idea/
app/build.gradle
View file @ 719611fb
...@@ -14,6 +14,10 @@ android { ...@@ -14,6 +14,10 @@ android {
versionCode 1000100 versionCode 1000100
versionName "1.0.1" versionName "1.0.1"
ndk {
abiFilters 'armeabi-v7a'
}
testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
javaCompileOptions { javaCompileOptions {
......
app/src/main/AndroidManifest.xml
View file @ 719611fb
...@@ -29,6 +29,7 @@ ...@@ -29,6 +29,7 @@
android:supportsRtl="true" android:supportsRtl="true"
android:theme="@style/Theme.CateringDetect" android:theme="@style/Theme.CateringDetect"
android:usesCleartextTraffic="true" android:usesCleartextTraffic="true"
android:hardwareAccelerated="true"
tools:targetApi="31"> tools:targetApi="31">
<activity <activity
android:name=".ui.SplashActivity" android:name=".ui.SplashActivity"
......
camera/src/main/java/com/wmdigit/camera/CameraxController.java
View file @ 719611fb
...@@ -3,13 +3,20 @@ package com.wmdigit.camera; ...@@ -3,13 +3,20 @@ package com.wmdigit.camera;
import android.annotation.SuppressLint; import android.annotation.SuppressLint;
import android.content.Context; import android.content.Context;
import android.graphics.Bitmap; import android.graphics.Bitmap;
import android.graphics.SurfaceTexture;
import android.hardware.camera2.CaptureRequest;
import android.opengl.GLES11Ext;
import android.util.Range;
import android.util.Size; import android.util.Size;
import androidx.camera.camera2.interop.Camera2Interop;
import androidx.camera.core.Camera; import androidx.camera.core.Camera;
import androidx.camera.core.CameraInfo; import androidx.camera.core.CameraInfo;
import androidx.camera.core.CameraSelector; import androidx.camera.core.CameraSelector;
import androidx.camera.core.ImageAnalysis; import androidx.camera.core.ImageAnalysis;
import androidx.camera.core.ImageProxy; import androidx.camera.core.ImageProxy;
import androidx.camera.core.Preview;
import androidx.camera.core.UseCaseGroup;
import androidx.camera.lifecycle.ProcessCameraProvider; import androidx.camera.lifecycle.ProcessCameraProvider;
import androidx.core.content.ContextCompat; import androidx.core.content.ContextCompat;
import androidx.lifecycle.LifecycleOwner; import androidx.lifecycle.LifecycleOwner;
...@@ -50,7 +57,7 @@ public class CameraxController { ...@@ -50,7 +57,7 @@ public class CameraxController {
/** /**
* 图片分析线程 * 图片分析线程
*/ */
private final ExecutorService executors = Executors.newFixedThreadPool(1); private final ExecutorService executors = Executors.newSingleThreadExecutor();
/** /**
* YUV转RGB转化器 * YUV转RGB转化器
*/ */
...@@ -60,6 +67,8 @@ public class CameraxController { ...@@ -60,6 +67,8 @@ public class CameraxController {
*/ */
private OnImageAnalyzeListener onImageAnalyzeListener; private OnImageAnalyzeListener onImageAnalyzeListener;
// private SurfaceTexture mColorSurfaceTexture = new SurfaceTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES);
public static CameraxController getInstance(Context context) { public static CameraxController getInstance(Context context) {
if (instance == null){ if (instance == null){
synchronized (CameraxController.class){ synchronized (CameraxController.class){
...@@ -124,6 +133,7 @@ public class CameraxController { ...@@ -124,6 +133,7 @@ public class CameraxController {
}, ContextCompat.getMainExecutor(context)); }, ContextCompat.getMainExecutor(context));
} }
int count = 0;
/** /**
* 创建图片分析器 * 创建图片分析器
* @return * @return
...@@ -140,11 +150,14 @@ public class CameraxController { ...@@ -140,11 +150,14 @@ public class CameraxController {
imageAnalysis.setAnalyzer(executors, imageProxy->{ imageAnalysis.setAnalyzer(executors, imageProxy->{
long startTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis();
// 处理图片 // 处理图片
analyzeImage(imageProxy); count ++;
if (count % 5 == 0) {
analyzeImage(imageProxy);
}
imageProxy.close(); imageProxy.close();
// 休眠 // 休眠
long costTime = System.currentTimeMillis() - startTime; /*long costTime = System.currentTimeMillis() - startTime;
System.out.println("处理时间：" + costTime); // System.out.println("处理时间：" + costTime);
long leftTime = INTERVAL_FRAMES_ANALYZE - costTime; long leftTime = INTERVAL_FRAMES_ANALYZE - costTime;
if (leftTime <= INTERVAL_FRAMES_ANALYZE && leftTime >= 0){ if (leftTime <= INTERVAL_FRAMES_ANALYZE && leftTime >= 0){
try { try {
...@@ -152,7 +165,7 @@ public class CameraxController { ...@@ -152,7 +165,7 @@ public class CameraxController {
} catch (InterruptedException e) { } catch (InterruptedException e) {
e.printStackTrace(); e.printStackTrace();
} }
} }*/
}); });
return imageAnalysis; return imageAnalysis;
......
common/src/main/java/com/wmdigit/common/utils/YuvToRgbConverter.kt 0 → 100644
View file @ 719611fb
package com.wmdigit.common.utils
import android.content.Context
import android.graphics.Bitmap
import android.graphics.ImageFormat
import android.graphics.Rect
import android.media.Image
import android.renderscript.Allocation
import android.renderscript.Element
import android.renderscript.RenderScript
import android.renderscript.ScriptIntrinsicYuvToRGB
import android.renderscript.Type
import java.nio.ByteBuffer
/**
* Helper class used to efficiently convert a [Media.Image] object from
* [ImageFormat.YUV_420_888] format to an RGB [Bitmap] object.
*
* The [yuvToRgb] method is able to achieve the same FPS as the CameraX image
* analysis use case on a Pixel 3 XL device at the default analyzer resolution,
* which is 30 FPS with 640x480.
*
* NOTE: This has been tested in a limited number of devices and is not
* considered production-ready code. It was created for illustration purposes,
* since this is not an efficient camera pipeline due to the multiple copies
* required to convert each frame.
*/
class YuvToRgbConverter(context: Context) {
private val rs = RenderScript.create(context)
private val scriptYuvToRgb = ScriptIntrinsicYuvToRGB.create(rs, Element.U8_4(rs))
private var pixelCount: Int = -1
private lateinit var yuvBuffer: ByteBuffer
private lateinit var inputAllocation: Allocation
private lateinit var outputAllocation: Allocation
@Synchronized
fun yuvToRgb(image: Image, output: Bitmap) {
// Ensure that the intermediate output byte buffer is allocated
if (!::yuvBuffer.isInitialized) {
pixelCount = image.cropRect.width() * image.cropRect.height()
// Bits per pixel is an average for the whole image, so it's useful to compute the size
// of the full buffer but should not be used to determine pixel offsets
val pixelSizeBits = ImageFormat.getBitsPerPixel(ImageFormat.YUV_420_888)
yuvBuffer = ByteBuffer.allocateDirect(pixelCount * pixelSizeBits / 8)
}
// Rewind the buffer; no need to clear it since it will be filled
yuvBuffer.rewind()
// Get the YUV data in byte array form using NV21 format
imageToByteBuffer(image, yuvBuffer.array())
// Ensure that the RenderScript inputs and outputs are allocated
if (!::inputAllocation.isInitialized) {
// Explicitly create an element with type NV21, since that's the pixel format we use
val elemType = Type.Builder(rs, Element.YUV(rs)).setYuvFormat(ImageFormat.NV21).create()
inputAllocation = Allocation.createSized(rs, elemType.element, yuvBuffer.array().size)
}
if (!::outputAllocation.isInitialized) {
outputAllocation = Allocation.createFromBitmap(rs, output)
}
// Convert NV21 format YUV to RGB
inputAllocation.copyFrom(yuvBuffer.array())
scriptYuvToRgb.setInput(inputAllocation)
scriptYuvToRgb.forEach(outputAllocation)
outputAllocation.copyTo(output)
}
private fun imageToByteBuffer(image: Image, outputBuffer: ByteArray) {
assert(image.format == ImageFormat.YUV_420_888)
val imageCrop = image.cropRect
val imagePlanes = image.planes
imagePlanes.forEachIndexed { planeIndex, plane ->
// How many values are read in input for each output value written
// Only the Y plane has a value for every pixel, U and V have half the resolution i.e.
//
// Y Plane U Plane V Plane
// =============== ======= =======
// Y Y Y Y Y Y Y Y U U U U V V V V
// Y Y Y Y Y Y Y Y U U U U V V V V
// Y Y Y Y Y Y Y Y U U U U V V V V
// Y Y Y Y Y Y Y Y U U U U V V V V
// Y Y Y Y Y Y Y Y
// Y Y Y Y Y Y Y Y
// Y Y Y Y Y Y Y Y
val outputStride: Int
// The index in the output buffer the next value will be written at
// For Y it's zero, for U and V we start at the end of Y and interleave them i.e.
//
// First chunk Second chunk
// =============== ===============
// Y Y Y Y Y Y Y Y V U V U V U V U
// Y Y Y Y Y Y Y Y V U V U V U V U
// Y Y Y Y Y Y Y Y V U V U V U V U
// Y Y Y Y Y Y Y Y V U V U V U V U
// Y Y Y Y Y Y Y Y
// Y Y Y Y Y Y Y Y
// Y Y Y Y Y Y Y Y
var outputOffset: Int
when (planeIndex) {
0 -> {
outputStride = 1
outputOffset = 0
}
1 -> {
outputStride = 2
// For NV21 format, U is in odd-numbered indices
outputOffset = pixelCount + 1
}
2 -> {
outputStride = 2
// For NV21 format, V is in even-numbered indices
outputOffset = pixelCount
}
else -> {
// Image contains more than 3 planes, something strange is going on
return@forEachIndexed
}
}
val planeBuffer = plane.buffer
val rowStride = plane.rowStride
val pixelStride = plane.pixelStride
// We have to divide the width and height by two if it's not the Y plane
val planeCrop = if (planeIndex == 0) {
imageCrop
} else {
Rect(
imageCrop.left / 2,
imageCrop.top / 2,
imageCrop.right / 2,
imageCrop.bottom / 2
)
}
val planeWidth = planeCrop.width()
val planeHeight = planeCrop.height()
// Intermediate buffer used to store the bytes of each row
val rowBuffer = ByteArray(plane.rowStride)
// Size of each row in bytes
val rowLength = if (pixelStride == 1 && outputStride == 1) {
planeWidth
} else {
// Take into account that the stride may include data from pixels other than this
// particular plane and row, and that could be between pixels and not after every
// pixel:
//
// |---- Pixel stride ----| Row ends here --> |
// | Pixel 1 | Other Data | Pixel 2 | Other Data | ... | Pixel N |
//
// We need to get (N-1) * (pixel stride bytes) per row + 1 byte for the last pixel
(planeWidth - 1) * pixelStride + 1
}
for (row in 0 until planeHeight) {
// Move buffer position to the beginning of this row
planeBuffer.position(
(row + planeCrop.top) * rowStride + planeCrop.left * pixelStride)
if (pixelStride == 1 && outputStride == 1) {
// When there is a single stride value for pixel and output, we can just copy
// the entire row in a single step
planeBuffer.get(outputBuffer, outputOffset, rowLength)
outputOffset += rowLength
} else {
// When either pixel or output have a stride > 1 we must copy pixel by pixel
planeBuffer.get(rowBuffer, 0, rowLength)
for (col in 0 until planeWidth) {
outputBuffer[outputOffset] = rowBuffer[col * pixelStride]
outputOffset += outputStride
}
}
}
}
}
}
\ No newline at end of file
core/build.gradle
View file @ 719611fb
...@@ -10,7 +10,7 @@ android { ...@@ -10,7 +10,7 @@ android {
minSdk 24 minSdk 24
ndk { ndk {
abiFilters 'armeabi-v7a', "arm64-v8a" abiFilters 'armeabi-v7a'
// abiFilters 'armeabi-v7a','arm64-v8a','x86','x86_64' // abiFilters 'armeabi-v7a','arm64-v8a','x86','x86_64'
} }
......
core/src/main/cpp/CMakeLists.txt
View file @ 719611fb
...@@ -54,6 +54,39 @@ set_target_properties( ...@@ -54,6 +54,39 @@ set_target_properties(
# ${CMAKE_SOURCE_DIR}/../jniLibs/${ANDROID_ABI}/libdetfea.a # ${CMAKE_SOURCE_DIR}/../jniLibs/${ANDROID_ABI}/libdetfea.a
#) #)
#视频流
add_library(videopipe STATIC IMPORTED)
set_target_properties(
videopipe
PROPERTIES IMPORTED_LOCATION
${CMAKE_SOURCE_DIR}/../jniLibs/${ANDROID_ABI}/libvideopipe.a
)
add_library(io3_lat3d_c3d STATIC IMPORTED)
set_target_properties(
io3_lat3d_c3d
PROPERTIES IMPORTED_LOCATION
${CMAKE_SOURCE_DIR}/../jniLibs/${ANDROID_ABI}/libio3_lat3d_c3d.a
)
add_library(io3_lat3d_backbone STATIC IMPORTED)
set_target_properties(
io3_lat3d_backbone
PROPERTIES IMPORTED_LOCATION
${CMAKE_SOURCE_DIR}/../jniLibs/${ANDROID_ABI}/libio3_lat3d_backbone.a
)
# rknn库
add_library(rknnapi STATIC IMPORTED)
set_target_properties(rknnapi PROPERTIES
IMPORTED_LOCATION
${CMAKE_SOURCE_DIR}/../jniLibs/${ANDROID_ABI}/librknn_api.so)
add_library(rknnrt STATIC IMPORTED)
set_target_properties(rknnrt PROPERTIES
IMPORTED_LOCATION
${CMAKE_SOURCE_DIR}/../jniLibs/${ANDROID_ABI}/librknnrt.so)
# Log库 # Log库
find_library( # Sets the name of the path variable. find_library( # Sets the name of the path variable.
log-lib log-lib
...@@ -79,6 +112,28 @@ target_link_libraries( ...@@ -79,6 +112,28 @@ target_link_libraries(
${log-lib} ${log-lib}
) )
# 视频流
add_library(
video_pipe
SHARED
videopipe_rk3568.cpp
)
target_link_libraries(
video_pipe
android
wmai
io3_lat3d_backbone
io3_lat3d_c3d
opencv
jnigraphics
c++_shared
videopipe
rknnrt
rknnapi
image_tools
${log-lib})
# 餐饮目标检测 # 餐饮目标检测
add_library( add_library(
......
core/src/main/cpp/image_tools.cpp
View file @ 719611fb
...@@ -37,11 +37,11 @@ cv::Mat convert_bitmap_to_mat(JNIEnv *env, jobject bitmap){ ...@@ -37,11 +37,11 @@ cv::Mat convert_bitmap_to_mat(JNIEnv *env, jobject bitmap){
// 根据Bitmap的图片格式决定type // 根据Bitmap的图片格式决定type
int type; int type;
if (info.format == ANDROID_BITMAP_FORMAT_RGBA_8888){ if (info.format == ANDROID_BITMAP_FORMAT_RGBA_8888){
LOGD("CV_8UC4"); // LOGD("CV_8UC4");
type = CV_8UC4; type = CV_8UC4;
} }
else if (info.format == ANDROID_BITMAP_FORMAT_RGB_565){ else if (info.format == ANDROID_BITMAP_FORMAT_RGB_565){
LOGD("CV_8UC2"); // LOGD("CV_8UC2");
type = CV_8UC2; type = CV_8UC2;
} }
else{ else{
......
core/src/main/cpp/include/rknn_api.h 0 → 100644
View file @ 719611fb
/****************************************************************************
*
* Copyright (c) 2017 - 2022 by Rockchip Corp. All rights reserved.
*
* The material in this file is confidential and contains trade secrets
* of Rockchip Corporation. This is proprietary information owned by
* Rockchip Corporation. No part of this work may be disclosed,
* reproduced, copied, transmitted, or used in any way for any purpose,
* without the express written permission of Rockchip Corporation.
*
*****************************************************************************/
#ifndef _RKNN_API_H
#define _RKNN_API_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
/*
Definition of extended flag for rknn_init.
*/
/* set high priority context. */
#define RKNN_FLAG_PRIOR_HIGH 0x00000000
/* set medium priority context */
#define RKNN_FLAG_PRIOR_MEDIUM 0x00000001
/* set low priority context. */
#define RKNN_FLAG_PRIOR_LOW 0x00000002
/* asynchronous mode.
when enable, rknn_outputs_get will not block for too long because it directly retrieves the result of
the previous frame which can increase the frame rate on single-threaded mode, but at the cost of
rknn_outputs_get not retrieves the result of the current frame.
in multi-threaded mode you do not need to turn this mode on. */
#define RKNN_FLAG_ASYNC_MASK 0x00000004
/* collect performance mode.
when enable, you can get detailed performance reports via rknn_query(ctx, RKNN_QUERY_PERF_DETAIL, ...),
but it will reduce the frame rate. */
#define RKNN_FLAG_COLLECT_PERF_MASK 0x00000008
/* allocate all memory in outside, includes weight/internal/inputs/outputs */
#define RKNN_FLAG_MEM_ALLOC_OUTSIDE 0x00000010
/* weight sharing with the same network structure */
#define RKNN_FLAG_SHARE_WEIGHT_MEM 0x00000020
/* send fence fd from outside */
#define RKNN_FLAG_FENCE_IN_OUTSIDE 0x00000040
/* get fence fd from inside */
#define RKNN_FLAG_FENCE_OUT_OUTSIDE 0x00000080
/*
Error code returned by the RKNN API.
*/
#define RKNN_SUCC 0 /* execute succeed. */
#define RKNN_ERR_FAIL -1 /* execute failed. */
#define RKNN_ERR_TIMEOUT -2 /* execute timeout. */
#define RKNN_ERR_DEVICE_UNAVAILABLE -3 /* device is unavailable. */
#define RKNN_ERR_MALLOC_FAIL -4 /* memory malloc fail. */
#define RKNN_ERR_PARAM_INVALID -5 /* parameter is invalid. */
#define RKNN_ERR_MODEL_INVALID -6 /* model is invalid. */
#define RKNN_ERR_CTX_INVALID -7 /* context is invalid. */
#define RKNN_ERR_INPUT_INVALID -8 /* input is invalid. */
#define RKNN_ERR_OUTPUT_INVALID -9 /* output is invalid. */
#define RKNN_ERR_DEVICE_UNMATCH -10 /* the device is unmatch, please update rknn sdk
and npu driver/firmware. */
#define RKNN_ERR_INCOMPATILE_PRE_COMPILE_MODEL -11 /* This RKNN model use pre_compile mode, but not compatible with current driver. */
#define RKNN_ERR_INCOMPATILE_OPTIMIZATION_LEVEL_VERSION -12 /* This RKNN model set optimization level, but not compatible with current driver. */
#define RKNN_ERR_TARGET_PLATFORM_UNMATCH -13 /* This RKNN model set target platform, but not compatible with current platform. */
/*
Definition for tensor
*/
#define RKNN_MAX_DIMS 16 /* maximum dimension of tensor. */
#define RKNN_MAX_NUM_CHANNEL 15 /* maximum channel number of input tensor. */
#define RKNN_MAX_NAME_LEN 256 /* maximum name lenth of tensor. */
#ifdef __arm__
typedef uint32_t rknn_context;
#else
typedef uint64_t rknn_context;
#endif
/*
The query command for rknn_query
*/
typedef enum _rknn_query_cmd {
RKNN_QUERY_IN_OUT_NUM = 0, /* query the number of input & output tensor. */
RKNN_QUERY_INPUT_ATTR = 1, /* query the attribute of input tensor. */
RKNN_QUERY_OUTPUT_ATTR = 2, /* query the attribute of output tensor. */
RKNN_QUERY_PERF_DETAIL = 3, /* query the detail performance, need set
RKNN_FLAG_COLLECT_PERF_MASK when call rknn_init,
this query needs to be valid after rknn_outputs_get. */
RKNN_QUERY_PERF_RUN = 4, /* query the time of run,
this query needs to be valid after rknn_outputs_get. */
RKNN_QUERY_SDK_VERSION = 5, /* query the sdk & driver version */
RKNN_QUERY_MEM_SIZE = 6, /* query the weight & internal memory size */
RKNN_QUERY_CUSTOM_STRING = 7, /* query the custom string */
RKNN_QUERY_NATIVE_INPUT_ATTR = 8, /* query the attribute of native input tensor. */
RKNN_QUERY_NATIVE_OUTPUT_ATTR = 9, /* query the attribute of native output tensor. */
RKNN_QUERY_NATIVE_NC1HWC2_INPUT_ATTR = 8, /* query the attribute of native input tensor. */
RKNN_QUERY_NATIVE_NC1HWC2_OUTPUT_ATTR = 9, /* query the attribute of native output tensor. */
RKNN_QUERY_NATIVE_NHWC_INPUT_ATTR = 10, /* query the attribute of native input tensor. */
RKNN_QUERY_NATIVE_NHWC_OUTPUT_ATTR = 11, /* query the attribute of native output tensor. */
RKNN_QUERY_DEVICE_MEM_INFO = 12, /* query the attribute of rknn memory information. */
RKNN_QUERY_CMD_MAX
} rknn_query_cmd;
/*
the tensor data type.
*/
typedef enum _rknn_tensor_type {
RKNN_TENSOR_FLOAT32 = 0, /* data type is float32. */
RKNN_TENSOR_FLOAT16, /* data type is float16. */
RKNN_TENSOR_INT8, /* data type is int8. */
RKNN_TENSOR_UINT8, /* data type is uint8. */
RKNN_TENSOR_INT16, /* data type is int16. */
RKNN_TENSOR_UINT16, /* data type is uint16. */
RKNN_TENSOR_INT32, /* data type is int32. */
RKNN_TENSOR_UINT32, /* data type is uint32. */
RKNN_TENSOR_INT64, /* data type is int64. */
RKNN_TENSOR_BOOL,
RKNN_TENSOR_TYPE_MAX
} rknn_tensor_type;
inline static const char* get_type_string(rknn_tensor_type type)
{
switch(type) {
case RKNN_TENSOR_FLOAT32: return "FP32";
case RKNN_TENSOR_FLOAT16: return "FP16";
case RKNN_TENSOR_INT8: return "INT8";
case RKNN_TENSOR_UINT8: return "UINT8";
case RKNN_TENSOR_INT16: return "INT16";
case RKNN_TENSOR_UINT16: return "UINT16";
case RKNN_TENSOR_INT32: return "INT32";
case RKNN_TENSOR_UINT32: return "UINT32";
case RKNN_TENSOR_INT64: return "INT64";
case RKNN_TENSOR_BOOL: return "BOOL";
default: return "UNKNOW";
}
}
/*
the quantitative type.
*/
typedef enum _rknn_tensor_qnt_type {
RKNN_TENSOR_QNT_NONE = 0, /* none. */
RKNN_TENSOR_QNT_DFP, /* dynamic fixed point. */
RKNN_TENSOR_QNT_AFFINE_ASYMMETRIC, /* asymmetric affine. */
RKNN_TENSOR_QNT_MAX
} rknn_tensor_qnt_type;
inline static const char* get_qnt_type_string(rknn_tensor_qnt_type type)
{
switch(type) {
case RKNN_TENSOR_QNT_NONE: return "NONE";
case RKNN_TENSOR_QNT_DFP: return "DFP";
case RKNN_TENSOR_QNT_AFFINE_ASYMMETRIC: return "AFFINE";
default: return "UNKNOW";
}
}
/*
the tensor data format.
*/
typedef enum _rknn_tensor_format {
RKNN_TENSOR_NCHW = 0, /* data format is NCHW. */
RKNN_TENSOR_NHWC, /* data format is NHWC. */
RKNN_TENSOR_NC1HWC2, /* data format is NC1HWC2. */
RKNN_TENSOR_UNDEFINED,
RKNN_TENSOR_FORMAT_MAX
} rknn_tensor_format;
/*
the mode of running on target NPU core.
*/
typedef enum _rknn_core_mask {
RKNN_NPU_CORE_AUTO = 0, /* default, run on NPU core randomly. */
RKNN_NPU_CORE_0 = 1, /* run on NPU core 0. */
RKNN_NPU_CORE_1 = 2, /* run on NPU core 1. */
RKNN_NPU_CORE_2 = 4, /* run on NPU core 2. */
RKNN_NPU_CORE_0_1 = RKNN_NPU_CORE_0 | RKNN_NPU_CORE_1, /* run on NPU core 1 and core 2. */
RKNN_NPU_CORE_0_1_2 = RKNN_NPU_CORE_0_1 | RKNN_NPU_CORE_2, /* run on NPU core 1 and core 2 and core 3. */
RKNN_NPU_CORE_UNDEFINED,
} rknn_core_mask;
inline static const char* get_format_string(rknn_tensor_format fmt)
{
switch(fmt) {
case RKNN_TENSOR_NCHW: return "NCHW";
case RKNN_TENSOR_NHWC: return "NHWC";
case RKNN_TENSOR_NC1HWC2: return "NC1HWC2";
case RKNN_TENSOR_UNDEFINED: return "UNDEFINED";
default: return "UNKNOW";
}
}
/*
the information for RKNN_QUERY_IN_OUT_NUM.
*/
typedef struct _rknn_input_output_num {
uint32_t n_input; /* the number of input. */
uint32_t n_output; /* the number of output. */
} rknn_input_output_num;
/*
the information for RKNN_QUERY_INPUT_ATTR / RKNN_QUERY_OUTPUT_ATTR.
*/
typedef struct _rknn_tensor_attr {
uint32_t index; /* input parameter, the index of input/output tensor,
need set before call rknn_query. */
uint32_t n_dims; /* the number of dimensions. */
uint32_t dims[RKNN_MAX_DIMS]; /* the dimensions array. */
char name[RKNN_MAX_NAME_LEN]; /* the name of tensor. */
uint32_t n_elems; /* the number of elements. */
uint32_t size; /* the bytes size of tensor. */
rknn_tensor_format fmt; /* the data format of tensor. */
rknn_tensor_type type; /* the data type of tensor. */
rknn_tensor_qnt_type qnt_type; /* the quantitative type of tensor. */
int8_t fl; /* fractional length for RKNN_TENSOR_QNT_DFP. */
int32_t zp; /* zero point for RKNN_TENSOR_QNT_AFFINE_ASYMMETRIC. */
float scale; /* scale for RKNN_TENSOR_QNT_AFFINE_ASYMMETRIC. */
uint32_t w_stride; /* the stride of tensor along the width dimention of input,
Note: it is read-only, 0 means equal to width. */
uint32_t size_with_stride; /* the bytes size of tensor with stride. */
uint8_t pass_through; /* pass through mode, for rknn_set_io_mem interface.
if TRUE, the buf data is passed directly to the input node of the rknn model
without any conversion. the following variables do not need to be set.
if FALSE, the buf data is converted into an input consistent with the model
according to the following type and fmt. so the following variables
need to be set.*/
uint32_t h_stride; /* the stride along the height dimention of input,
Note: it is write-only, if it was set to 0, h_stride = height. */
} rknn_tensor_attr;
/*
the information for RKNN_QUERY_PERF_DETAIL.
*/
typedef struct _rknn_perf_detail {
char* perf_data; /* the string pointer of perf detail. don't need free it by user. */
uint64_t data_len; /* the string length. */
} rknn_perf_detail;
/*
the information for RKNN_QUERY_PERF_RUN.
*/
typedef struct _rknn_perf_run {
int64_t run_duration; /* real inference time (us) */
} rknn_perf_run;
/*
the information for RKNN_QUERY_SDK_VERSION.
*/
typedef struct _rknn_sdk_version {
char api_version[256]; /* the version of rknn api. */
char drv_version[256]; /* the version of rknn driver. */
} rknn_sdk_version;
/*
the information for RKNN_QUERY_MEM_SIZE.
*/
typedef struct _rknn_mem_size {
uint32_t total_weight_size; /* the weight memory size */
uint32_t total_internal_size; /* the internal memory size, exclude inputs/outputs */
uint64_t total_dma_allocated_size; /* total dma memory allocated size */
uint32_t total_sram_size; /* total system sram size reserved for rknn */
uint32_t free_sram_size; /* free system sram size reserved for rknn */
uint32_t reserved[10]; /* reserved */
} rknn_mem_size;
/*
the information for RKNN_QUERY_CUSTOM_STRING.
*/
typedef struct _rknn_custom_string {
char string[1024]; /* the string of custom, lengths max to 1024 bytes */
} rknn_custom_string;
/*
The flags of rknn_tensor_mem.
*/
typedef enum _rknn_tensor_mem_flags {
RKNN_TENSOR_MEMORY_FLAGS_ALLOC_INSIDE = 1, /*Used to mark in rknn_destroy_mem() whether it is necessary to release the "mem" pointer itself.
If the flag RKNN_TENSOR_MEMORY_FLAGS_ALLOC_INSIDE is set, rknn_destroy_mem() will call free(mem).*/
RKNN_TENSOR_MEMORY_FLAGS_FROM_FD = 2, /*Used to mark in rknn_create_mem_from_fd() whether it is necessary to release the "mem" pointer itself.
If the flag RKNN_TENSOR_MEMORY_FLAGS_FROM_FD is set, rknn_destroy_mem() will call free(mem).*/
RKNN_TENSOR_MEMORY_FLAGS_FROM_PHYS = 3, /*Used to mark in rknn_create_mem_from_phys() whether it is necessary to release the "mem" pointer itself.
If the flag RKNN_TENSOR_MEMORY_FLAGS_FROM_PHYS is set, rknn_destroy_mem() will call free(mem).*/
RKNN_TENSOR_MEMORY_FLAGS_UNKNOWN
} rknn_tensor_mem_flags;
/*
the memory information of tensor.
*/
typedef struct _rknn_tensor_memory {
void* virt_addr; /* the virtual address of tensor buffer. */
uint64_t phys_addr; /* the physical address of tensor buffer. */
int32_t fd; /* the fd of tensor buffer. */
int32_t offset; /* indicates the offset of the memory. */
uint32_t size; /* the size of tensor buffer. */
uint32_t flags; /* the flags of tensor buffer, reserved */
void * priv_data; /* the private data of tensor buffer. */
} rknn_tensor_mem;
/*
the input information for rknn_input_set.
*/
typedef struct _rknn_input {
uint32_t index; /* the input index. */
void* buf; /* the input buf for index. */
uint32_t size; /* the size of input buf. */
uint8_t pass_through; /* pass through mode.
if TRUE, the buf data is passed directly to the input node of the rknn model
without any conversion. the following variables do not need to be set.
if FALSE, the buf data is converted into an input consistent with the model
according to the following type and fmt. so the following variables
need to be set.*/
rknn_tensor_type type; /* the data type of input buf. */
rknn_tensor_format fmt; /* the data format of input buf.
currently the internal input format of NPU is NCHW by default.
so entering NCHW data can avoid the format conversion in the driver. */
} rknn_input;
/*
the output information for rknn_outputs_get.
*/
typedef struct _rknn_output {
uint8_t want_float; /* want transfer output data to float */
uint8_t is_prealloc; /* whether buf is pre-allocated.
if TRUE, the following variables need to be set.
if FALSE, the following variables do not need to be set. */
uint32_t index; /* the output index. */
void* buf; /* the output buf for index.
when is_prealloc = FALSE and rknn_outputs_release called,
this buf pointer will be free and don't use it anymore. */
uint32_t size; /* the size of output buf. */
} rknn_output;
/*
the extend information for rknn_init.
*/
typedef struct _rknn_init_extend {
rknn_context ctx; /* rknn context */
int32_t real_model_offset; /* real rknn model file size, only valid when init context with rknn file path */
uint32_t real_model_size; /* real rknn model file offset, only valid when init context with rknn file path */
uint8_t reserved[120]; /* reserved */
} rknn_init_extend;
/*
the extend information for rknn_run.
*/
typedef struct _rknn_run_extend {
uint64_t frame_id; /* output parameter, indicate current frame id of run. */
int32_t non_block; /* block flag of run, 0 is block else 1 is non block */
int32_t timeout_ms; /* timeout for block mode, in milliseconds */
int32_t fence_fd; /* fence fd from other unit */
} rknn_run_extend;
/*
the extend information for rknn_outputs_get.
*/
typedef struct _rknn_output_extend {
uint64_t frame_id; /* output parameter, indicate the frame id of outputs, corresponds to
struct rknn_run_extend.frame_id.*/
} rknn_output_extend;
/* rknn_init
initial the context and load the rknn model.
input:
rknn_context* context the pointer of context handle.
void* model if size > 0, pointer to the rknn model, if size = 0, filepath to the rknn model.
uint32_t size the size of rknn model.
uint32_t flag extend flag, see the define of RKNN_FLAG_XXX_XXX.
rknn_init_extend* extend the extend information of init.
return:
int error code.
*/
int rknn_init(rknn_context* context, void* model, uint32_t size, uint32_t flag, rknn_init_extend* extend);
/* rknn_dup_context
initial the context and load the rknn model.
input:
rknn_context* context_in the pointer of context in handle.
rknn_context* context_out the pointer of context out handle.
return:
int error code.
*/
int rknn_dup_context(rknn_context* context_in, rknn_context* context_out);
/* rknn_destroy
unload the rknn model and destroy the context.
input:
rknn_context context the handle of context.
return:
int error code.
*/
int rknn_destroy(rknn_context context);
/* rknn_query
query the information about model or others. see rknn_query_cmd.
input:
rknn_context context the handle of context.
rknn_query_cmd cmd the command of query.
void* info the buffer point of information.
uint32_t size the size of information.
return:
int error code.
*/
int rknn_query(rknn_context context, rknn_query_cmd cmd, void* info, uint32_t size);
/* rknn_inputs_set
set inputs information by input index of rknn model.
inputs information see rknn_input.
input:
rknn_context context the handle of context.
uint32_t n_inputs the number of inputs.
rknn_input inputs[] the arrays of inputs information, see rknn_input.
return:
int error code
*/
int rknn_inputs_set(rknn_context context, uint32_t n_inputs, rknn_input inputs[]);
/*
rknn_set_batch_core_num
set rknn batch core_num.
input:
rknn_context context the handle of context.
int core_num the core number.
return:
int error code.
*/
int rknn_set_batch_core_num(rknn_context context, int core_num);
/* rknn_set_core_mask
set rknn core mask.(only supported on RK3588 now)
RKNN_NPU_CORE_AUTO: auto mode, default value
RKNN_NPU_CORE_0: core 0 mode
RKNN_NPU_CORE_1: core 1 mode
RKNN_NPU_CORE_2: core 2 mode
RKNN_NPU_CORE_0_1: combine core 0/1 mode
RKNN_NPU_CORE_0_1_2: combine core 0/1/2 mode
input:
rknn_context context the handle of context.
rknn_core_mask core_mask the core mask.
return:
int error code.
*/
int rknn_set_core_mask(rknn_context context, rknn_core_mask core_mask);
/* rknn_run
run the model to execute inference.
input:
rknn_context context the handle of context.
rknn_run_extend* extend the extend information of run.
return:
int error code.
*/
int rknn_run(rknn_context context, rknn_run_extend* extend);
/* rknn_wait
wait the model after execute inference.
input:
rknn_context context the handle of context.
rknn_run_extend* extend the extend information of run.
return:
int error code.
*/
int rknn_wait(rknn_context context, rknn_run_extend* extend);
/* rknn_outputs_get
wait the inference to finish and get the outputs.
this function will block until inference finish.
the results will set to outputs[].
input:
rknn_context context the handle of context.
uint32_t n_outputs the number of outputs.
rknn_output outputs[] the arrays of output, see rknn_output.
rknn_output_extend* the extend information of output.
return:
int error code.
*/
int rknn_outputs_get(rknn_context context, uint32_t n_outputs, rknn_output outputs[], rknn_output_extend* extend);
/* rknn_outputs_release
release the outputs that get by rknn_outputs_get.
after called, the rknn_output[x].buf get from rknn_outputs_get will
also be free when rknn_output[x].is_prealloc = FALSE.
input:
rknn_context context the handle of context.
uint32_t n_ouputs the number of outputs.
rknn_output outputs[] the arrays of output.
return:
int error code
*/
int rknn_outputs_release(rknn_context context, uint32_t n_ouputs, rknn_output outputs[]);
/* new api for zero copy */
/* rknn_create_mem_from_phys (memory allocated outside)
initialize tensor memory from physical address.
input:
rknn_context ctx the handle of context.
uint64_t phys_addr physical address.
void *virt_addr virtual address.
uint32_t size the size of tensor buffer.
return:
rknn_tensor_mem the pointer of tensor memory information.
*/
rknn_tensor_mem* rknn_create_mem_from_phys(rknn_context ctx, uint64_t phys_addr, void *virt_addr, uint32_t size);
/* rknn_create_mem_from_fd (memory allocated outside)
initialize tensor memory from file description.
input:
rknn_context ctx the handle of context.
int32_t fd file description.
void *virt_addr virtual address.
uint32_t size the size of tensor buffer.
int32_t offset indicates the offset of the memory (virt_addr without offset).
return:
rknn_tensor_mem the pointer of tensor memory information.
*/
rknn_tensor_mem* rknn_create_mem_from_fd(rknn_context ctx, int32_t fd, void *virt_addr, uint32_t size, int32_t offset);
/* rknn_create_mem_from_mb_blk (memory allocated outside)
create tensor memory from mb_blk.
input:
rknn_context ctx the handle of context.
void *mb_blk mb_blk allocate from system api.
int32_t offset indicates the offset of the memory.
return:
rknn_tensor_mem the pointer of tensor memory information.
*/
rknn_tensor_mem* rknn_create_mem_from_mb_blk(rknn_context ctx, void *mb_blk, int32_t offset);
/* rknn_create_mem (memory allocated inside)
create tensor memory.
input:
rknn_context ctx the handle of context.
uint32_t size the size of tensor buffer.
return:
rknn_tensor_mem the pointer of tensor memory information.
*/
rknn_tensor_mem* rknn_create_mem(rknn_context ctx, uint32_t size);
/* rknn_destroy_mem (support allocate inside and outside)
destroy tensor memory.
input:
rknn_context ctx the handle of context.
rknn_tensor_mem *mem the pointer of tensor memory information.
return:
int error code
*/
int rknn_destroy_mem(rknn_context ctx, rknn_tensor_mem *mem);
/* rknn_set_weight_mem
set the weight memory.
input:
rknn_context ctx the handle of context.
rknn_tensor_mem *mem the array of tensor memory information
return:
int error code.
*/
int rknn_set_weight_mem(rknn_context ctx, rknn_tensor_mem *mem);
/* rknn_set_internal_mem
set the internal memory.
input:
rknn_context ctx the handle of context.
rknn_tensor_mem *mem the array of tensor memory information
return:
int error code.
*/
int rknn_set_internal_mem(rknn_context ctx, rknn_tensor_mem *mem);
/* rknn_set_io_mem
set the input and output tensors buffer.
input:
rknn_context ctx the handle of context.
rknn_tensor_mem *mem the array of tensor memory information.
rknn_tensor_attr *attr the attribute of input or output tensor buffer.
return:
int error code.
*/
int rknn_set_io_mem(rknn_context ctx, rknn_tensor_mem *mem, rknn_tensor_attr *attr);
#ifdef __cplusplus
} //extern "C"
#endif
#endif //_RKNN_API_H
core/src/main/cpp/include/videopipe.h 0 → 100644
View file @ 719611fb
#ifndef _VIDEOPIPE_H_
#define _VIDEOPIPE_H_
#include <vector>
#include <map>
#include <string>
#include <opencv2/core.hpp>
namespace libvideopipe
{
#define WM_AI_VIDEO_PIPE_OK 0x00000000 // OK
#define WM_AI_VIDEO_FOOD_PIPE_MARK_EMPTY 0x10020001 // 输入temp为空
#define WM_AI_VIDEO_FOOD_PIPE_INPUT_FRAME_EMPTY 0x10020002 // 输入input数据为空
#define WM_AI_VIDEO_FOOD_PIPE_INPUT_FRAME_SIZE_SMALLER_THAN_MARK 0x10020003 // 输入input数据尺寸比mark小
#define WM_AI_VIDEO_FOOD_PIPE_ORB_TEMP_POINTS_ZERO 0x10020004
#define WM_AI_VIDEO_MOTION_PIPE_INPUT_FRAME_EMPTY 0x10020101
#define WM_AI_VIDEO_MOTION_PIPE_INIT_PARAM_ERROR 0x10020102
#define WM_AI_VIDEO_MOTION_PIPE_HANDLE_EMPTY 0x10020103
#define WM_AI_VIDEO_MOTION_PIPE_MARK_EMPTY 0x10020104
#define WM_AI_VIDEO_DEPTH_PIPE_INPUT_FRAME_EMPTY 0x10020201
#define WM_AI_VIDEO_DEPTH_PIPE_INPUT_PARAM_ERROR 0x10020202
// 视频流回调函数申明
typedef void (*VideoPipeCallback)(const char* data, int len, void* user);
struct INIT_INFO
{
bool enable_ai = false; // 是否启动ai匹配判断 FOOD
bool gpu_enable = false; // 是否开启gpu FOOD Motion
bool lat3d_enable = true; // 是否开启lat3d算法 Motion
int match_type = 0; // 0: 边缘 1：orb
unsigned char * lat3d_backbone_model_data = NULL;
int lat3d_backbone_model_len;
unsigned char * lat3d_c3d_model_data = NULL;
int lat3d_c3d_model_len;
};
//*****************************************
//food
//*****************************************
struct FOOD_VIDEOPIPE_INPUT
{
unsigned long int pts; // 时间戳
cv::Mat img; // 输入的该帧图片(roi之后)
bool debug_mode; // 是否debug模式
};
enum FOOD_REALTIME_SERVICE_PLATE_STATUS_TYPES
{
STATIC_EMPTY = 0, // 静止的空盘
MOVING_EMPTY = 1, // 运动的空盘
STATIC_FULL = 2, // 静止的餐盘
MOVING_FULL = 3, // 运动的餐盘
MOVING_2_STATIC_EMPTY = 4, // 运动->静止空
STATIC_EMPTY_2_MOVING = 5, // 静止空->运动
STATIC_FULL_2_MOVING = 6, // 静止满->运动
MOVING_2_STATIC_FULL = 7, // 运动->静止满
};
struct DEBUG_INFO //此部分为调试信息
{
cv::Mat back, fore, shadow;
float movement_area;
float match;
};
struct ALG_PARAM
{
cv::Mat mark_mat;
float mark_threshold;
float mark_ai_threshold;
float move_lr_threshold;
float move_area_threshold;
float shadow_threshold;
float var_threshold;
};
enum FOOD_STATIC_FULL_TYPES // 输出的类型
{
NORMAL = 0, // 正常输出
CHANGED = 1, // 经过更改后的输出
};
struct FOOD_VIDEOPIPE_OUTPUT
{
int index; // 图像索引，目前该字段无效
bool has_best_obj; // 是否具有目标，如果有则需要后续的AI识别
unsigned long int output_pts; // 时间戳（与输入时的时间戳一致）
cv::Mat best_img; // 最佳的用于识别的图像帧
bool debug_mode; // 是否debug模式
FOOD_STATIC_FULL_TYPES output_types; // 用于判断是否二次更改
FOOD_REALTIME_SERVICE_PLATE_STATUS_TYPES service_plate_status; // 输出状态类型
DEBUG_INFO debug_info; // debug信息
int error_code; // 错误码
};
class FoodVideoPipe
{
public:
FoodVideoPipe(INIT_INFO init_info);
~FoodVideoPipe();
public:
// 开始追踪判断目标
int Trace(FOOD_VIDEOPIPE_INPUT input);
// 开始Lat3D追踪判断目标
int TraceLat3D(FOOD_VIDEOPIPE_INPUT input);
// 设置回调函数
void SetResultCallback(VideoPipeCallback cb, void* user);
// 目前该函数无效
void SetROI(cv::Rect roi);
// 设置标志物的图像
void SetMarkMat(cv::Mat temp);
// 返回标志物的图像
cv::Mat GetMarkMat();
// 设置标志物的阈值，这个阈值范围是0~1(建议0.95~0.995调调看)；如果值设越大，越容易“把空盘当餐盘”回调出来。
void SetMarkThreshold(float threshold);
float GetMarkThreshold();
// 设置标志物的AI阈值，这个阈值范围是0~1(建议0.90~0.93调调看)；要想这个值有效，则必须enable_ai=true。
void SetAIMarkThreshold(float threshold);
float GetAIMarkThreshold();
// 设置运动程度学习率的阈值，这个阈值范围是0.001~0.3；值设得越小，静止状态越来得慢
void SetMovementThreshold(float threshold);
float GetMovementThreshold();
// 阴影检测开关
void SetShadowEnable(bool enable);
bool GetShadowEnable();
// 阴影检测的阈值，设得越小，越容易把前景判断为阴影 [0.005~1] 默认是：0.05
void SetShadowThreshold(float threshold);
float GetShadowThreshold();
// 前景和背景对比度阈值，设得越小，越容易前景检测灵敏 [1~1000] 默认是：500
void SetVarThreshold(float threshold);
float GetVarThreshold();
// 设置运动面积的阈值：0.01~0.5值越小，运动检测触发得灵敏些，稳定性差些
void SetAreaThreshold(float threshold);
float GetAreaThreshold();
// 设置模板的缩放系数，一般根据一次trace的耗时长短来定
void SetTempScale(float scale);
float GetTempScale();
// 获取匹配的结果
float GetMatchResult();
private:
void trace_out(FOOD_VIDEOPIPE_INPUT input);
bool movement_detect(FOOD_VIDEOPIPE_INPUT input);
bool mark_detect(FOOD_VIDEOPIPE_INPUT input);
private:
// roi
cv::Rect roi_;
// call back
VideoPipeCallback cb_function_;
void* cb_param_;
// video track handle
void* video_handle_;
};
//*****************************************
//motion
//*****************************************
struct MOTION_VIDEOPIPE_INPUT
{
unsigned long int pts; // 时间戳
cv::Mat img; // 输入的该帧图片(roi之后)
bool debug_mode; // 是否debug模式;
int debug_dump_img_type; // -1:all 0: 只存empty 1: 只存full 2:只存in 3: 只存out 4: 只存empty->full 5: 只存full->empty
};
// 'empty': 0, 'full': 1,'in': 2, 'out': 3
enum MOTION_STATUS_TYPES
{
MOTION_STATIC_EMPTY = 0, // 静止的空
MOTION_STATIC_FULL = 1, // 静止的满
MOTION_IN = 2, // 进入
MOTION_OUT = 3, // 离开
};
struct MOTION_VIDEOPIPE_OUTPUT
{
int index; // 图像索引，目前该字段无效
unsigned long int output_pts; // 时间戳（与输入时的时间戳一致）
cv::Mat best_img; // 最佳的用于识别的图像帧
MOTION_STATUS_TYPES status; // 状态
bool debug_mode; // 是否debug模式
int error_code; // 错误码
};
class MotionVideoPipe
{
public:
MotionVideoPipe();
~MotionVideoPipe();
public:
int Init(INIT_INFO init_info);
void SetROI(cv::Rect roi);
// 喂数据
int FeedFrame(MOTION_VIDEOPIPE_INPUT input);
// 设置回调函数
void SetResultCallback(VideoPipeCallback cb, void* user);
// 设置标志物的图像
int SetMarkMat(cv::Mat temp);
// 返回标志物的图像
cv::Mat GetMarkMat();
private:
void trace_out(MOTION_VIDEOPIPE_INPUT input);
private:
// roi
cv::Rect roi_;
// call back
VideoPipeCallback cb_function_;
void* cb_param_;
//
void* video_handle_;
};
//*****************************************
//depth
//*****************************************
enum DEPTH_VIDEOPIPE_STATUS // 输出的类型
{
DEPTH_IN = 0, // 进入
DEPTH_OUT = 1, // 出去
};
struct DEPTH_VIDEOPIPE_INPUT
{
unsigned long int pts; // 时间戳
cv::Mat img; // 输入的该帧图片(roi之后)
bool debug_mode; // 是否debug模式
};
struct DEPTH_VIDEOPIPE_OUTPUT
{
int index; // 图像索引，目前该字段无效
unsigned long int output_pts; // 当前时刻帧的时间戳（与输入时的时间戳一致）
DEPTH_VIDEOPIPE_STATUS status; // 状态
cv::Mat best_img; // 最佳的图像帧
float best_area; // 最佳的面积
unsigned long int best_pts; // 最佳的图像帧的时间戳
bool debug_mode; // 是否debug模式
int error_code; // 错误码
};
class DepthVideoPipe
{
public:
DepthVideoPipe();
~DepthVideoPipe();
public:
void SetROI(cv::Rect roi);
// 喂数据
int FeedFrame(DEPTH_VIDEOPIPE_INPUT input);
// 设置有效深度信息检测范围，这个阈值范围；
int SetVaildDepthRange(float* valid_depth_range);
int GetVaildDepthRange(float* valid_depth_range);
// 设置回调函数
void SetResultCallback(VideoPipeCallback cb, void* user);
// 设置运动程度的阈值，这个阈值范围是0~1(建议0.1~0.2调调看)；
// 值设得越小，检测运动越灵敏
void SetAreaThreshold(float threshold);
float GetAreaThreshold();
// 设置运动程度的阈值，这个阈值范围是0~1(建议0.1~0.2调调看)；
// 值设得越小，检测进入越灵敏
void SetInThreshold(float threshold);
float GetInThreshold();
// 设置运动程度的阈值，这个阈值范围是0~1(建议0.1~0.2调调看)；
// 值设得越大，检测出去越灵敏
void SetOutThreshold(float threshold);
float GetOutThreshold();
private:
void trace_out(DEPTH_VIDEOPIPE_STATUS status,DEPTH_VIDEOPIPE_INPUT input);
bool detect_obj(DEPTH_VIDEOPIPE_INPUT input);
private:
// roi
cv::Rect roi_;
// call back
VideoPipeCallback cb_function_;
void* cb_param_;
//
void* video_handle_;
};
}
#endif
core/src/main/cpp/include/videopipe_rk3568.h 0 → 100644
View file @ 719611fb
//
// Created by dizi on 2024/8/2.
//
#ifndef CATERINGDETECT_VIDEOPIPE_RK3568_H
#define CATERINGDETECT_VIDEOPIPE_RK3568_H
#include <jni.h>
#include "image_tools.h"
#include "videopipe.h"
libvideopipe::MotionVideoPipe* motion;
void* user;
libvideopipe::INIT_INFO init_info;
JavaVM *global_jvm;
// VideoPipeManager实例
jclass java_class_video_pipe_manager;
long long get_timestamp();
void feed_frame_callback(int state);
void trace_result_callback(const char* data, int len, void* user);
unsigned char* read_file_byte(JNIEnv *env, jstring file_path, int& size);
#endif //CATERINGDETECT_VIDEOPIPE_RK3568_H
core/src/main/cpp/videopipe_rk3568.cpp 0 → 100644
View file @ 719611fb
//
// Created by dizi on 2024/8/2.
//
#include "videopipe_rk3568.h"
JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *reserved) {
global_jvm = vm;
JNIEnv *env;
if (vm->GetEnv(reinterpret_cast<void**>(&env), JNI_VERSION_1_6) != JNI_OK) {
return JNI_ERR;
}
java_class_video_pipe_manager = (jclass)env->NewGlobalRef(env->FindClass("com/wmdigit/core/videopipe/VideoPipeManager"));
return JNI_VERSION_1_6;
}
/**
* 算法初始化
*/
extern "C"
JNIEXPORT jint JNICALL
Java_com_wmdigit_core_videopipe_VideoPipeManager_init(JNIEnv *env, jobject thiz,
jstring modelBackBonePath,
jstring modelC3dPath) {
int size_back_bone, size_c3d;
unsigned char* data_back_bone = read_file_byte(env, modelBackBonePath, size_back_bone);
unsigned char* data_c3d = read_file_byte(env, modelC3dPath, size_c3d);
init_info.lat3d_backbone_model_data = data_back_bone;
init_info.lat3d_backbone_model_len = size_back_bone;
init_info.lat3d_c3d_model_data = data_c3d;
init_info.lat3d_c3d_model_len = size_c3d;
init_info.lat3d_enable = true;
init_info.gpu_enable = true;
motion = new libvideopipe::MotionVideoPipe();
// 初始化
int ret_init = motion->Init(init_info);
// 设置回调
if (ret_init == 0){
libvideopipe::VideoPipeCallback callback = trace_result_callback;
motion->SetResultCallback(callback, &user);
}
return ret_init;
}
/**
* 配置空盘图
*/
extern "C"
JNIEXPORT jint JNICALL
Java_com_wmdigit_core_videopipe_VideoPipeManager_setMarkMat(JNIEnv *env, jobject thiz,
jobject bitmap) {
// Bitmap转mat
cv::Mat mat = convert_bitmap_to_mat(env, bitmap);
// RGBA转RGB
cv::Mat mat_rgb = cv::Mat();
cv::cvtColor(mat, mat_rgb, CV_RGBA2RGB);
// 设置空盘图
return motion->SetMarkMat(mat_rgb.clone());
}
extern "C"
JNIEXPORT void JNICALL
Java_com_wmdigit_core_videopipe_VideoPipeManager_feedFrame(JNIEnv *env, jobject thiz,
jobject bitmap) {
// Bitmap转mat
cv::Mat mat = convert_bitmap_to_mat(env, bitmap);
// rgba转rgb
cv::Mat mat_rgb = cv::Mat();
cv::cvtColor(mat, mat_rgb, CV_RGBA2RGB);
// 组装入参
libvideopipe::MOTION_VIDEOPIPE_INPUT input;
input.img = mat_rgb.clone();
input.pts = get_timestamp();
input.debug_mode = false;
motion->FeedFrame(input);
}
/**
* FeedFrame的回调
* @param data
* @param len
* @param user
*/
void trace_result_callback(const char* data, int len, void* user)
{
auto* output = (libvideopipe::MOTION_VIDEOPIPE_OUTPUT*) data;
if (output->status == libvideopipe::MOTION_STATUS_TYPES::MOTION_IN)
{
LOGD("进入");
feed_frame_callback(2);
}
else if (output->status == libvideopipe::MOTION_STATUS_TYPES::MOTION_OUT)
{
LOGD("离开");
feed_frame_callback(3);
}
else if (output->status == libvideopipe::MOTION_STATUS_TYPES::MOTION_STATIC_EMPTY)
{
}
else if (output->status == libvideopipe::MOTION_STATUS_TYPES::MOTION_STATIC_FULL)
{
}
else{
}
}
void feed_frame_callback(int state){
JNIEnv *env;
global_jvm->GetEnv((void**)&env, JNI_VERSION_1_6);
jmethodID j_method_instance = env->GetStaticMethodID(
java_class_video_pipe_manager,
"getInstance",
"()Lcom/wmdigit/core/videopipe/VideoPipeManager;"
);
jobject instance=env->CallStaticObjectMethod(java_class_video_pipe_manager, j_method_instance);
jmethodID j_method_in_out;
if (state == 2){
j_method_in_out = env->GetMethodID(java_class_video_pipe_manager, "objectIn", "()V");
}
else{
j_method_in_out = env->GetMethodID(java_class_video_pipe_manager, "objectOut", "()V");
}
env->CallVoidMethod(instance, j_method_in_out);
env->DeleteLocalRef(instance);
}
/**
* 读本地文件，用于加载模型
* @param env
* @param file_path
* @return
*/
unsigned char* read_file_byte(JNIEnv *env, jstring file_path, int& size){
const char *jpath = env->GetStringUTFChars(file_path, nullptr);
FILE *pFile;
pFile = fopen(jpath, "rw");
if (pFile == nullptr) {
env->ReleaseStringUTFChars(file_path, jpath);
return nullptr;
}
unsigned char *pBuf; //定义文件指针
fseek(pFile, 0, SEEK_END); //把指针移动到文件的结尾 ，获取文件长度
size = (int) ftell(pFile); //获取文件长度
LOGD("size:%d", size);
pBuf = new unsigned char[size + 1]; //定义数组长度
rewind(pFile); //把指针移动到文件开头 因为我们一开始把指针移动到结尾，如果不移动回来 会出错
fread(pBuf, 1, (size_t) size, pFile); //读文件
pBuf[size] = 0; //把读到的文件最后一位 写为0 要不然系统会一直寻找到0后才结束
fclose(pFile); // 关闭文件
env->ReleaseStringUTFChars(file_path, jpath);
return pBuf;
}
/**
* 获取时间戳
* @return
*/
long long get_timestamp()
{
long long tmp;
struct timeval tv;
gettimeofday(&tv, nullptr);
tmp = tv.tv_sec;
tmp = tmp * 1000;
tmp = tmp + (tv.tv_usec / 1000);
return tmp;
}
\ No newline at end of file
core/src/main/java/com/wmdigit/core/CoreModule.java
View file @ 719611fb
...@@ -3,6 +3,7 @@ package com.wmdigit.core; ...@@ -3,6 +3,7 @@ package com.wmdigit.core;
import android.content.Context; import android.content.Context;
import com.wmdigit.core.catering.TargetDetectionManager; import com.wmdigit.core.catering.TargetDetectionManager;
import com.wmdigit.core.videopipe.VideoPipeManager;
/** /**
* Core模块初始化类 * Core模块初始化类
...@@ -26,7 +27,10 @@ public class CoreModule { ...@@ -26,7 +27,10 @@ public class CoreModule {
* 初始化算法库 * 初始化算法库
*/ */
private static void initCore() { private static void initCore() {
// 初始化目标检测
TargetDetectionManager.getInstance().initTargetDetection(); TargetDetectionManager.getInstance().initTargetDetection();
// 初始化视频流
VideoPipeManager.getInstance().initVideoPipe();
} }
public static Context getAppContext() { public static Context getAppContext() {
......
core/src/main/java/com/wmdigit/core/videopipe/VideoPipeManager.java 0 → 100644
View file @ 719611fb
package com.wmdigit.core.videopipe;
import android.graphics.Bitmap;
import com.elvishew.xlog.XLog;
import com.wmdigit.common.utils.FileUtils;
import com.wmdigit.core.CoreModule;
import java.io.File;
import java.io.IOException;
/**
* 视频流管理类
* @author dizi
*/
public class VideoPipeManager {
static {
System.loadLibrary("video_pipe");
}
/**
* 初始化模型
* @param modelBackBonePath
* @param modelC3dPath
* @return
*/
private native int init(String modelBackBonePath, String modelC3dPath);
/**
* 设置空盘图
* @param bitmap
*/
private native int setMarkMat(Bitmap bitmap);
/**
* 喂图片帧
* @param bitmap
*/
private native void feedFrame(Bitmap bitmap);
private static VideoPipeManager instance;
public static VideoPipeManager getInstance() {
if (instance == null){
synchronized (VideoPipeManager.class){
if (instance == null){
instance = new VideoPipeManager();
}
}
}
return instance;
}
/**
* 初始化视频流算法
*/
public void initVideoPipe(){
// 拷贝模型文件
String modelRootPath = CoreModule.getAppContext().getExternalFilesDir("model").getAbsolutePath();
String modelBackBonePath = modelRootPath + "/lat3d_backbone.rknn";
String modelC3dPath = modelRootPath + "/lat3d_c3d.rknn";
File fileBackBone = new File(modelBackBonePath);
File fileC3d = new File(modelC3dPath);
// 检查本地是否有模型文件
if (!fileBackBone.exists() || !fileC3d.exists()){
// 将assert模型文件输出到disk
try {
FileUtils.copyAssertsFolderToDisk(CoreModule.getAppContext(), "model", modelRootPath);
} catch (IOException e) {
XLog.e(e.toString());
}
}
// 初始化模型
int ret = init(modelBackBonePath, modelC3dPath);
XLog.i("视频流初始化结果：" + ret);
}
/**
* 设置空盘
* @param bitmap
*/
public void setEmptyImage(Bitmap bitmap){
int ret = setMarkMat(bitmap);
XLog.i("设置空盘结果：" + ret);
}
/**
* 喂图片帧
* @param bitmap
*/
public void processImage(Bitmap bitmap){
feedFrame(bitmap);
}
public void objectIn(){
}
public void objectOut(){
}
}
history.txt
View file @ 719611fb
v1.0.0 2024/07/18 1.演示Demo v1.0.0 2024/07/18 1.演示Demo
v1.0.1 2024/08/01 1.增加camerax v1.0.1 2024/08/01 1.增加camerax
2.todo 增加3568视频流算法 2.增加3568视频流算法
\ No newline at end of file \ No newline at end of file
module-demo/src/main/java/com/wmdigit/cateringdetect/demo/ui/fragment/DemoHomeFragment.java
View file @ 719611fb
...@@ -14,6 +14,7 @@ import com.wmdigit.cateringdetect.demo.ui.adapter.DemoImagesAdapter; ...@@ -14,6 +14,7 @@ import com.wmdigit.cateringdetect.demo.ui.adapter.DemoImagesAdapter;
import com.wmdigit.cateringdetect.demo.ui.adapter.ShoppingCartAdapter; import com.wmdigit.cateringdetect.demo.ui.adapter.ShoppingCartAdapter;
import com.wmdigit.cateringdetect.demo.ui.viewmodel.DemoHomeViewModel; import com.wmdigit.cateringdetect.demo.ui.viewmodel.DemoHomeViewModel;
import com.wmdigit.common.base.mvvm.BaseMvvmFragment; import com.wmdigit.common.base.mvvm.BaseMvvmFragment;
import com.wmdigit.core.videopipe.VideoPipeManager;
/** /**
...@@ -88,7 +89,19 @@ public class DemoHomeFragment extends BaseMvvmFragment<DemoHomeViewModel, Fragme ...@@ -88,7 +89,19 @@ public class DemoHomeFragment extends BaseMvvmFragment<DemoHomeViewModel, Fragme
.apply(); .apply();
} }
int count = 0;
private void processBitmap(Bitmap bitmap){ private void processBitmap(Bitmap bitmap){
if (count < 10) {
count++;
}
if (count == 10){
// 设置背景
VideoPipeManager.getInstance().setEmptyImage(bitmap);
count ++;
}
else{
VideoPipeManager.getInstance().processImage(bitmap);
}
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment