fixed a bug

This commit is contained in:
2026-08-07 16:06:43 +08:00
parent ddfb57c02c
commit c79c34895c
8 changed files with 470 additions and 75 deletions

View File

@@ -8,10 +8,10 @@ import androidx.camera.core.ImageProxy
@ExperimentalGetImage @ExperimentalGetImage
object ImageUtils { object ImageUtils {
fun imageProxyToBitmap(imageProxy: ImageProxy): Bitmap { fun imageProxyToBitmap(imageProxy: ImageProxy, applyRotation: Boolean = true): Bitmap {
val image = imageProxy.image ?: error("ImageProxy does not contain an image.") val image = imageProxy.image ?: error("ImageProxy does not contain an image.")
val bitmap = yuv420ToBitmap(image) val bitmap = yuv420ToBitmap(image)
if (imageProxy.imageInfo.rotationDegrees == 0) { if (!applyRotation || imageProxy.imageInfo.rotationDegrees == 0) {
return bitmap return bitmap
} }
val matrix = Matrix().apply { val matrix = Matrix().apply {

View File

@@ -42,6 +42,7 @@ import com.example.studentfaceregistry.face.MatchType
import com.example.studentfaceregistry.face.RecognitionResult import com.example.studentfaceregistry.face.RecognitionResult
import com.google.mlkit.vision.common.InputImage import com.google.mlkit.vision.common.InputImage
import com.google.mlkit.vision.face.Face import com.google.mlkit.vision.face.Face
import com.google.mlkit.vision.face.FaceLandmark
import com.example.studentfaceregistry.ui.DetectionUi import com.example.studentfaceregistry.ui.DetectionUi
import com.example.studentfaceregistry.ui.OverlayView import com.example.studentfaceregistry.ui.OverlayView
import com.example.studentfaceregistry.upload.UploadServer import com.example.studentfaceregistry.upload.UploadServer
@@ -115,6 +116,20 @@ class MainActivity : AppCompatActivity() {
repository.students.collectLatest { list -> repository.students.collectLatest { list ->
students = list students = list
countText.text = getString(R.string.registered_student_count, list.size) countText.text = getString(R.string.registered_student_count, list.size)
if (list.isNotEmpty()) {
val first = list.first()
val earliest = list.minOfOrNull { it.createdAt } ?: 0L
val latest = list.maxOfOrNull { it.createdAt } ?: 0L
Log.i(
TAG,
"已加载学生 ${list.size} 人," +
"特征维度=${first.embedding.size}" +
"注册时间范围=${fmtTime(earliest)} ~ ${fmtTime(latest)}" +
"特征范数=${String.format(java.util.Locale.US, "%.3f", embeddingNorm(first.embedding))}"
)
} else {
Log.i(TAG, "已加载学生 0 人(请先注册)")
}
} }
} }
@@ -589,8 +604,18 @@ class MainActivity : AppCompatActivity() {
if (facesNeedingRefresh.isNotEmpty()) { if (facesNeedingRefresh.isNotEmpty()) {
val refreshStart = SystemClock.elapsedRealtime() val refreshStart = SystemClock.elapsedRealtime()
val bitmap = ImageUtils.imageProxyToBitmap(image) val bitmap = ImageUtils.imageProxyToBitmap(image)
var unrotatedBitmap: Bitmap? = null
facesNeedingRefresh.forEach { (index, face) -> facesNeedingRefresh.forEach { (index, face) ->
val refreshed = refreshRecognition(faceProcessor, bitmap, face) var refreshed = refreshRecognition(faceProcessor, bitmap, face)
if (refreshed.matchType == MatchType.NO_MATCH && image.imageInfo.rotationDegrees != 0) {
val fallbackBitmap = unrotatedBitmap ?: ImageUtils
.imageProxyToBitmap(image, applyRotation = false)
.also { unrotatedBitmap = it }
val fallback = refreshRecognition(faceProcessor, fallbackBitmap, face)
if (recognitionScore(fallback) > recognitionScore(refreshed)) {
refreshed = fallback
}
}
refreshedResults[index] = refreshed refreshedResults[index] = refreshed
} }
val refreshMs = SystemClock.elapsedRealtime() - refreshStart val refreshMs = SystemClock.elapsedRealtime() - refreshStart
@@ -941,6 +966,13 @@ class MainActivity : AppCompatActivity() {
bitmap: Bitmap, bitmap: Bitmap,
face: Face face: Face
): RecognitionResult { ): RecognitionResult {
val landmarkCount = listOf(
face.getLandmark(FaceLandmark.LEFT_EYE),
face.getLandmark(FaceLandmark.RIGHT_EYE),
face.getLandmark(FaceLandmark.NOSE_BASE),
face.getLandmark(FaceLandmark.MOUTH_LEFT),
face.getLandmark(FaceLandmark.MOUTH_RIGHT)
).count { it != null }
val primary = matcher.findNearest(faceProcessor.embedFace(bitmap, face), students) val primary = matcher.findNearest(faceProcessor.embedFace(bitmap, face), students)
val cached = face.trackingId?.let { recognitionCache[it] } val cached = face.trackingId?.let { recognitionCache[it] }
// 未匹配时只做单次推理;仅在疑似匹配或人脸明显移动时才用多裁剪升级, // 未匹配时只做单次推理;仅在疑似匹配或人脸明显移动时才用多裁剪升级,
@@ -950,6 +982,7 @@ class MainActivity : AppCompatActivity() {
primary.matchType != MatchType.NO_MATCH primary.matchType != MatchType.NO_MATCH
if (!needsEscalation) { if (!needsEscalation) {
logRecognitionDebug("仅主候选", landmarkCount, primary)
return primary return primary
} }
@@ -960,9 +993,39 @@ class MainActivity : AppCompatActivity() {
best = result best = result
} }
} }
logRecognitionDebug("含升级候选", landmarkCount, primary, best)
return best return best
} }
private fun logRecognitionDebug(
stage: String,
landmarkCount: Int,
primary: RecognitionResult,
best: RecognitionResult = primary
) {
val primaryName = primary.student?.let { "${it.name}(${it.studentNo})" } ?: ""
val bestName = best.student?.let { "${it.name}(${it.studentNo})" } ?: ""
Log.i(
TAG,
String.format(
java.util.Locale.US,
"识别 debug: 阶段=%s 关键点=%d/5 主候选=%s(余弦=%.3f 欧氏=%.3f %s) " +
"最优=%s(余弦=%.3f 欧氏=%.3f %s) 学生=%d",
stage,
landmarkCount,
primaryName,
primary.cosineSimilarity,
primary.distance,
primary.matchType,
bestName,
best.cosineSimilarity,
best.distance,
best.matchType,
students.size
)
)
}
private fun resolveRecognitionResult( private fun resolveRecognitionResult(
face: Face, face: Face,
refreshedResult: RecognitionResult?, refreshedResult: RecognitionResult?,
@@ -1020,7 +1083,21 @@ class MainActivity : AppCompatActivity() {
} }
} }
private fun embeddingNorm(embedding: FloatArray): Double {
var sum = 0.0
for (value in embedding) {
sum += value.toDouble() * value.toDouble()
}
return kotlin.math.sqrt(sum)
}
private fun fmtTime(timestamp: Long): String {
return java.text.SimpleDateFormat("yyyy-MM-dd HH:mm", java.util.Locale.US)
.format(java.util.Date(timestamp))
}
companion object { companion object {
private const val TAG = "MainActivity"
private const val MATCHER_PREFS_NAME = "face_matcher_config" private const val MATCHER_PREFS_NAME = "face_matcher_config"
private const val KEY_EUCLIDEAN_THRESHOLD = "euclidean_threshold" private const val KEY_EUCLIDEAN_THRESHOLD = "euclidean_threshold"
private const val KEY_COSINE_THRESHOLD = "cosine_threshold" private const val KEY_COSINE_THRESHOLD = "cosine_threshold"

View File

@@ -13,7 +13,15 @@ class StudentRepository(context: Context) {
val students: StateFlow<List<Student>> = _students val students: StateFlow<List<Student>> = _students
fun add(student: Student) { fun add(student: Student) {
val next = _students.value.toMutableList().apply { add(student) } val studentToStore = if (student.id == 0L) {
student.copy(id = stableId(student.studentNo))
} else {
student
}
val next = _students.value
.filterNot { it.studentNo == studentToStore.studentNo }
.toMutableList()
.apply { add(studentToStore) }
.sortedByDescending { it.createdAt } .sortedByDescending { it.createdAt }
_students.value = next _students.value = next
save(next) save(next)
@@ -32,10 +40,11 @@ class StudentRepository(context: Context) {
return buildList { return buildList {
for (index in 0 until array.length()) { for (index in 0 until array.length()) {
val obj = array.getJSONObject(index) val obj = array.getJSONObject(index)
val studentNo = obj.getString("studentNo")
add( add(
Student( Student(
id = obj.optLong("id", index.toLong()), id = obj.optLong("id").takeIf { it != 0L } ?: stableId(studentNo),
studentNo = obj.getString("studentNo"), studentNo = studentNo,
name = obj.getString("name"), name = obj.getString("name"),
photoUri = obj.getString("photoUri"), photoUri = obj.getString("photoUri"),
embedding = decodeEmbedding(obj.getJSONArray("embedding")), embedding = decodeEmbedding(obj.getJSONArray("embedding")),
@@ -80,4 +89,8 @@ class StudentRepository(context: Context) {
} }
} }
} }
private fun stableId(studentNo: String): Long {
return (studentNo.hashCode().toLong() and 0xffffffffL).coerceAtLeast(1L)
}
} }

View File

@@ -1,11 +1,10 @@
package com.example.studentfaceregistry.face package com.example.studentfaceregistry.face
import android.graphics.Bitmap import android.graphics.Bitmap
import android.graphics.Canvas
import android.graphics.Matrix import android.graphics.Matrix
import android.graphics.Paint
import kotlin.math.absoluteValue import kotlin.math.absoluteValue
import kotlin.math.cos
import kotlin.math.sin
import kotlin.math.sqrt
/** /**
* 人脸对齐器 * 人脸对齐器
@@ -34,10 +33,18 @@ class FaceAligner {
faceTop: Int faceTop: Int
): Bitmap { ): Bitmap {
// 计算眼睛在裁剪后图片中的相对位置 // 计算眼睛在裁剪后图片中的相对位置
val leftEyeRelX = leftEyeX - faceLeft var leftEyeRelX = leftEyeX - faceLeft
val leftEyeRelY = leftEyeY - faceTop var leftEyeRelY = leftEyeY - faceTop
val rightEyeRelX = rightEyeX - faceLeft var rightEyeRelX = rightEyeX - faceLeft
val rightEyeRelY = rightEyeY - faceTop var rightEyeRelY = rightEyeY - faceTop
if (leftEyeRelX > rightEyeRelX) {
val oldLeftX = leftEyeRelX
val oldLeftY = leftEyeRelY
leftEyeRelX = rightEyeRelX
leftEyeRelY = rightEyeRelY
rightEyeRelX = oldLeftX
rightEyeRelY = oldLeftY
}
// 计算两眼连线的旋转角度(目标是让连线水平) // 计算两眼连线的旋转角度(目标是让连线水平)
val angle = calculateRotationAngle(leftEyeRelX, leftEyeRelY, rightEyeRelX, rightEyeRelY) val angle = calculateRotationAngle(leftEyeRelX, leftEyeRelY, rightEyeRelX, rightEyeRelY)
@@ -58,6 +65,40 @@ class FaceAligner {
return Bitmap.createBitmap(bitmap, 0, 0, bitmap.width, bitmap.height, matrix, true) return Bitmap.createBitmap(bitmap, 0, 0, bitmap.width, bitmap.height, matrix, true)
} }
fun alignFivePoint(
bitmap: Bitmap,
leftEyeX: Float,
leftEyeY: Float,
rightEyeX: Float,
rightEyeY: Float,
noseX: Float,
noseY: Float,
leftMouthX: Float,
leftMouthY: Float,
rightMouthX: Float,
rightMouthY: Float,
outputSize: Int = ARC_FACE_SIZE
): Bitmap {
val (imageLeftEye, imageRightEye) = sortedByX(leftEyeX, leftEyeY, rightEyeX, rightEyeY)
val (imageLeftMouth, imageRightMouth) = sortedByX(leftMouthX, leftMouthY, rightMouthX, rightMouthY)
val src = floatArrayOf(
imageLeftEye.x, imageLeftEye.y,
imageRightEye.x, imageRightEye.y,
noseX, noseY,
imageLeftMouth.x, imageLeftMouth.y,
imageRightMouth.x, imageRightMouth.y
)
val dst = arcFaceTemplate(outputSize)
val matrix = similarityTransform(src, dst)
val result = Bitmap.createBitmap(outputSize, outputSize, Bitmap.Config.ARGB_8888)
Canvas(result).drawBitmap(
bitmap,
matrix,
Paint(Paint.ANTI_ALIAS_FLAG or Paint.FILTER_BITMAP_FLAG)
)
return result
}
/** /**
* 计算两眼连线的旋转角度 * 计算两眼连线的旋转角度
*/ */
@@ -67,6 +108,63 @@ class FaceAligner {
return Math.atan2(dy.toDouble(), dx.toDouble()).toFloat() * 180f / Math.PI.toFloat() return Math.atan2(dy.toDouble(), dx.toDouble()).toFloat() * 180f / Math.PI.toFloat()
} }
private fun sortedByX(x1: Float, y1: Float, x2: Float, y2: Float): Pair<Point, Point> {
val first = Point(x1, y1)
val second = Point(x2, y2)
return if (x1 <= x2) first to second else second to first
}
private fun arcFaceTemplate(outputSize: Int): FloatArray {
val scale = outputSize / ARC_FACE_SIZE.toFloat()
return FloatArray(ARC_FACE_TEMPLATE.size) { index -> ARC_FACE_TEMPLATE[index] * scale }
}
private fun similarityTransform(src: FloatArray, dst: FloatArray): Matrix {
val count = src.size / 2
var srcCenterX = 0f
var srcCenterY = 0f
var dstCenterX = 0f
var dstCenterY = 0f
for (index in 0 until count) {
srcCenterX += src[index * 2]
srcCenterY += src[index * 2 + 1]
dstCenterX += dst[index * 2]
dstCenterY += dst[index * 2 + 1]
}
srcCenterX /= count
srcCenterY /= count
dstCenterX /= count
dstCenterY /= count
var numeratorA = 0f
var numeratorB = 0f
var denominator = 0f
for (index in 0 until count) {
val srcX = src[index * 2] - srcCenterX
val srcY = src[index * 2 + 1] - srcCenterY
val dstX = dst[index * 2] - dstCenterX
val dstY = dst[index * 2 + 1] - dstCenterY
numeratorA += srcX * dstX + srcY * dstY
numeratorB += srcX * dstY - srcY * dstX
denominator += srcX * srcX + srcY * srcY
}
val a = numeratorA / denominator.coerceAtLeast(1e-6f)
val b = numeratorB / denominator.coerceAtLeast(1e-6f)
val translateX = dstCenterX - a * srcCenterX + b * srcCenterY
val translateY = dstCenterY - b * srcCenterX - a * srcCenterY
return Matrix().apply {
setValues(
floatArrayOf(
a, -b, translateX,
b, a, translateY,
0f, 0f, 1f
)
)
}
}
/** /**
* 简单版本:仅根据估计的眼睛位置进行对齐 * 简单版本:仅根据估计的眼睛位置进行对齐
* 适用于没有启用关键点检测的场景 * 适用于没有启用关键点检测的场景
@@ -94,4 +192,17 @@ class FaceAligner {
align(bitmap, leftEyeX, eyeY, rightEyeX, eyeY, faceLeft, faceTop) align(bitmap, leftEyeX, eyeY, rightEyeX, eyeY, faceLeft, faceTop)
} }
} }
private data class Point(val x: Float, val y: Float)
companion object {
private const val ARC_FACE_SIZE = 112
private val ARC_FACE_TEMPLATE = floatArrayOf(
38.2946f, 51.6963f,
73.5318f, 51.5014f,
56.0252f, 71.7366f,
41.5493f, 92.3655f,
70.7299f, 92.2041f
)
}
} }

View File

@@ -30,6 +30,7 @@ class FaceEmbedder(context: Context, private val modelType: ModelType = ModelTyp
private val accelerator: Accelerator private val accelerator: Accelerator
private val inputPixels: IntArray private val inputPixels: IntArray
private val inputValues: FloatArray private val inputValues: FloatArray
private var debugEmbedCount = 0
// ArcFace 标准化参数 (RGB) // ArcFace 标准化参数 (RGB)
private val mean = floatArrayOf(127.5f, 127.5f, 127.5f) private val mean = floatArrayOf(127.5f, 127.5f, 127.5f)
@@ -85,12 +86,57 @@ class FaceEmbedder(context: Context, private val modelType: ModelType = ModelTyp
} }
} }
debugEmbedCount++
val logInput = debugEmbedCount <= 30 || debugEmbedCount % 100 == 0
if (logInput) {
var sum = 0f
var sumSq = 0f
for (value in inputValues) {
sum += value
sumSq += value * value
}
val mean = sum / inputValues.size
val variance = (sumSq / inputValues.size - mean * mean).coerceAtLeast(0f)
Log.i(
TAG,
String.format(
java.util.Locale.US,
"embed debug #%d 输入均值=%.3f 输入标准差=%.3f",
debugEmbedCount,
mean,
sqrt(variance)
)
)
}
// 执行推理 // 执行推理
inputBuffer.writeFloat(inputValues) inputBuffer.writeFloat(inputValues)
model.run(listOf(inputBuffer), listOf(outputBuffer)) model.run(listOf(inputBuffer), listOf(outputBuffer))
val rawOutput = outputBuffer.readFloat()
if (logInput) {
var sum = 0f
var sumSq = 0f
for (value in rawOutput) {
sum += value
sumSq += value * value
}
val mean = sum / rawOutput.size
val variance = (sumSq / rawOutput.size - mean * mean).coerceAtLeast(0f)
Log.i(
TAG,
String.format(
java.util.Locale.US,
"embed debug #%d 输出均值=%.3f 输出标准差=%.3f",
debugEmbedCount,
mean,
sqrt(variance)
)
)
}
// L2 归一化输出特征 // L2 归一化输出特征
return l2Normalize(outputBuffer.readFloat()) return l2Normalize(rawOutput)
} }
fun warmUp() { fun warmUp() {
@@ -138,19 +184,10 @@ class FaceEmbedder(context: Context, private val modelType: ModelType = ModelTyp
} }
private fun createModel(context: Context): Pair<CompiledModel, Accelerator> { private fun createModel(context: Context): Pair<CompiledModel, Accelerator> {
return runCatching { return CompiledModel.create(
CompiledModel.create( context.assets,
context.assets, MODEL_ASSET_NAME,
MODEL_ASSET_NAME, CompiledModel.Options(setOf(Accelerator.CPU))
CompiledModel.Options(setOf(Accelerator.GPU)) ) to Accelerator.CPU
) to Accelerator.GPU
}.getOrElse { gpuError ->
Log.w(TAG, "GPU initialization failed, falling back to CPU", gpuError)
CompiledModel.create(
context.assets,
MODEL_ASSET_NAME,
CompiledModel.Options(setOf(Accelerator.CPU))
) to Accelerator.CPU
}
} }
} }

View File

@@ -57,17 +57,19 @@ class FaceProcessor(context: Context) : AutoCloseable {
box.width() * box.height() box.width() * box.height()
} }
// 裁剪人脸 val alignedFace = alignFaceFromOriginal(bitmap, largestFace) ?: run {
val croppedFace = Bitmap.createBitmap( // 裁剪人脸
bitmap, val croppedFace = Bitmap.createBitmap(
faceBox.left, bitmap,
faceBox.top, faceBox.left,
faceBox.width(), faceBox.top,
faceBox.height() faceBox.width(),
) faceBox.height()
)
// 使用关键点进行对齐 // 使用关键点进行对齐
val alignedFace = alignFaceWithLandmarks(croppedFace, largestFace, faceBox.left, faceBox.top) alignFaceWithLandmarks(croppedFace, largestFace, faceBox.left, faceBox.top)
}
// 图像预处理 // 图像预处理
val preprocessedFace = preprocessor.preprocess(alignedFace) val preprocessedFace = preprocessor.preprocess(alignedFace)
@@ -127,6 +129,12 @@ class FaceProcessor(context: Context) : AutoCloseable {
* 提取指定人脸的特征 * 提取指定人脸的特征
*/ */
fun embedFace(bitmap: Bitmap, face: Face): FloatArray { fun embedFace(bitmap: Bitmap, face: Face): FloatArray {
val alignedFromLandmarks = alignFaceFromOriginal(bitmap, face)
if (alignedFromLandmarks != null) {
val preprocessed = preprocessor.preprocessRealtime(alignedFromLandmarks)
return embedder.embed(preprocessed)
}
val crop = requireNotNull(safeCropRect(bitmap, face.boundingBox)) { "人脸区域超出图片边界。" } val crop = requireNotNull(safeCropRect(bitmap, face.boundingBox)) { "人脸区域超出图片边界。" }
val cropped = Bitmap.createBitmap(bitmap, crop.left, crop.top, crop.width(), crop.height()) val cropped = Bitmap.createBitmap(bitmap, crop.left, crop.top, crop.width(), crop.height())
val aligned = alignFaceForRecognition(cropped, face, crop.left, crop.top) val aligned = alignFaceForRecognition(cropped, face, crop.left, crop.top)
@@ -136,6 +144,9 @@ class FaceProcessor(context: Context) : AutoCloseable {
fun embedFaceCandidates(bitmap: Bitmap, face: Face): List<FloatArray> { fun embedFaceCandidates(bitmap: Bitmap, face: Face): List<FloatArray> {
val embeddings = mutableListOf<FloatArray>() val embeddings = mutableListOf<FloatArray>()
alignFaceFromOriginal(bitmap, face)?.let { aligned ->
embeddings.add(embedder.embed(preprocessor.preprocessRealtime(aligned)))
}
for (paddingRatio in FACE_PADDING_RATIOS) { for (paddingRatio in FACE_PADDING_RATIOS) {
val crop = safeCropRect(bitmap, face.boundingBox, paddingRatio) ?: continue val crop = safeCropRect(bitmap, face.boundingBox, paddingRatio) ?: continue
val cropped = Bitmap.createBitmap(bitmap, crop.left, crop.top, crop.width(), crop.height()) val cropped = Bitmap.createBitmap(bitmap, crop.left, crop.top, crop.width(), crop.height())
@@ -146,6 +157,28 @@ class FaceProcessor(context: Context) : AutoCloseable {
return embeddings return embeddings
} }
private fun alignFaceFromOriginal(bitmap: Bitmap, face: Face): Bitmap? {
val leftEye = face.getLandmark(FaceLandmark.LEFT_EYE) ?: return null
val rightEye = face.getLandmark(FaceLandmark.RIGHT_EYE) ?: return null
val nose = face.getLandmark(FaceLandmark.NOSE_BASE) ?: return null
val leftMouth = face.getLandmark(FaceLandmark.MOUTH_LEFT) ?: return null
val rightMouth = face.getLandmark(FaceLandmark.MOUTH_RIGHT) ?: return null
return aligner.alignFivePoint(
bitmap = bitmap,
leftEyeX = leftEye.position.x,
leftEyeY = leftEye.position.y,
rightEyeX = rightEye.position.x,
rightEyeY = rightEye.position.y,
noseX = nose.position.x,
noseY = nose.position.y,
leftMouthX = leftMouth.position.x,
leftMouthY = leftMouth.position.y,
rightMouthX = rightMouth.position.x,
rightMouthY = rightMouth.position.y
)
}
fun warmUp() { fun warmUp() {
embedder.warmUp() embedder.warmUp()
} }

View File

@@ -101,20 +101,23 @@ class SmartEnrollment(context: Context) : AutoCloseable {
return EnrollmentResult.Failure("照片模糊,请换更清晰的照片。") return EnrollmentResult.Failure("照片模糊,请换更清晰的照片。")
} }
val embeddings = mutableListOf<FloatArray>() val alignedFromLandmarks = alignFaceFromOriginal(bitmap, largestFace.face)
for (paddingRatio in IMAGE_PADDING_RATIOS) { val cropEmbeddings = cropEmbeddingCandidates(bitmap, largestFace.face)
val cropRect = safeCropRect(bitmap, largestFace.face.boundingBox, paddingRatio) ?: continue if (alignedFromLandmarks != null) {
val croppedFace = cropFace(bitmap, cropRect) val preprocessedFace = preprocessor.preprocess(alignedFromLandmarks)
val alignedFace = alignFace(croppedFace, largestFace.face, cropRect) val embedding = embedder.embed(preprocessedFace)
val preprocessedFace = preprocessor.preprocess(alignedFace) return EnrollmentResult.Success(
embeddings.add(embedder.embed(preprocessedFace)) embedding,
EnrollmentSourceType.IMAGE,
extraEmbeddings = cropEmbeddings
)
} }
if (embeddings.isEmpty()) { if (cropEmbeddings.isEmpty()) {
return EnrollmentResult.Failure("未检测到可用人脸,请换更清晰或更正面的照片。") return EnrollmentResult.Failure("未检测到可用人脸,请换更清晰或更正面的照片。")
} }
val embedding = fuseEmbeddings(embeddings) val embedding = fuseEmbeddings(cropEmbeddings)
EnrollmentResult.Success(embedding, EnrollmentSourceType.IMAGE) EnrollmentResult.Success(embedding, EnrollmentSourceType.IMAGE)
} catch (e: Exception) { } catch (e: Exception) {
@@ -355,12 +358,30 @@ class SmartEnrollment(context: Context) : AutoCloseable {
* 对单帧提取特征(裁剪 + 对齐 + 预处理 + 推理) * 对单帧提取特征(裁剪 + 对齐 + 预处理 + 推理)
*/ */
private fun embedFrame(frameInfo: FaceFrameInfo): FloatArray { private fun embedFrame(frameInfo: FaceFrameInfo): FloatArray {
val alignedFromLandmarks = alignFaceFromOriginal(frameInfo.bitmap, frameInfo.face)
if (alignedFromLandmarks != null) {
val preprocessed = preprocessor.preprocess(alignedFromLandmarks)
return embedder.embed(preprocessed)
}
val cropped = cropFace(frameInfo.bitmap, frameInfo.cropRect) val cropped = cropFace(frameInfo.bitmap, frameInfo.cropRect)
val aligned = alignFace(cropped, frameInfo.face, frameInfo.cropRect) val aligned = alignFace(cropped, frameInfo.face, frameInfo.cropRect)
val preprocessed = preprocessor.preprocess(aligned) val preprocessed = preprocessor.preprocess(aligned)
return embedder.embed(preprocessed) return embedder.embed(preprocessed)
} }
private fun cropEmbeddingCandidates(bitmap: Bitmap, face: Face): List<FloatArray> {
return buildList {
for (paddingRatio in IMAGE_PADDING_RATIOS) {
val cropRect = safeCropRect(bitmap, face.boundingBox, paddingRatio) ?: continue
val croppedFace = cropFace(bitmap, cropRect)
val alignedFace = alignFace(croppedFace, face, cropRect)
val preprocessedFace = preprocessor.preprocess(alignedFace)
add(embedder.embed(preprocessedFace))
}
}
}
/** /**
* 按偏航角将帧特征分组:正脸(主特征)、左右侧脸(补充特征) * 按偏航角将帧特征分组:正脸(主特征)、左右侧脸(补充特征)
*/ */
@@ -471,6 +492,28 @@ class SmartEnrollment(context: Context) : AutoCloseable {
) )
} }
private fun alignFaceFromOriginal(bitmap: Bitmap, face: Face): Bitmap? {
val leftEye = face.getLandmark(FaceLandmark.LEFT_EYE) ?: return null
val rightEye = face.getLandmark(FaceLandmark.RIGHT_EYE) ?: return null
val nose = face.getLandmark(FaceLandmark.NOSE_BASE) ?: return null
val leftMouth = face.getLandmark(FaceLandmark.MOUTH_LEFT) ?: return null
val rightMouth = face.getLandmark(FaceLandmark.MOUTH_RIGHT) ?: return null
return aligner.alignFivePoint(
bitmap = bitmap,
leftEyeX = leftEye.position.x,
leftEyeY = leftEye.position.y,
rightEyeX = rightEye.position.x,
rightEyeY = rightEye.position.y,
noseX = nose.position.x,
noseY = nose.position.y,
leftMouthX = leftMouth.position.x,
leftMouthY = leftMouth.position.y,
rightMouthX = rightMouth.position.x,
rightMouthY = rightMouth.position.y
)
}
/** /**
* 解码 Bitmap * 解码 Bitmap
*/ */
@@ -522,7 +565,7 @@ class SmartEnrollment(context: Context) : AutoCloseable {
private val IMAGE_PADDING_RATIOS = floatArrayOf(0.18f, 0.25f, 0.34f) private val IMAGE_PADDING_RATIOS = floatArrayOf(0.18f, 0.25f, 0.34f)
private const val MIN_ENROLL_FACE_PX = 96 private const val MIN_ENROLL_FACE_PX = 96
private const val MAX_ENROLL_YAW = 30f private const val MAX_ENROLL_YAW = 30f
private const val MAX_ENROLL_PITCH = 15f private const val MAX_ENROLL_PITCH = 25f
private const val BLUR_VARIANCE_THRESHOLD = 40.0 private const val BLUR_VARIANCE_THRESHOLD = 40.0
} }
} }

View File

@@ -2,10 +2,12 @@ package com.example.studentfaceregistry.upload
import android.content.Context import android.content.Context
import android.util.Base64 import android.util.Base64
import android.util.Log
import com.example.studentfaceregistry.data.Student import com.example.studentfaceregistry.data.Student
import com.example.studentfaceregistry.data.StudentRepository import com.example.studentfaceregistry.data.StudentRepository
import com.example.studentfaceregistry.face.SmartEnrollment import com.example.studentfaceregistry.face.SmartEnrollment
import kotlinx.coroutines.runBlocking import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withTimeoutOrNull
import org.json.JSONObject import org.json.JSONObject
import java.io.BufferedInputStream import java.io.BufferedInputStream
import java.io.ByteArrayOutputStream import java.io.ByteArrayOutputStream
@@ -58,13 +60,18 @@ class UploadServer(
private fun handle(socket: Socket) { private fun handle(socket: Socket) {
thread(name = "upload-request", isDaemon = true) { thread(name = "upload-request", isDaemon = true) {
socket.use { client -> socket.use { client ->
val input = BufferedInputStream(client.getInputStream()) client.soTimeout = SOCKET_READ_TIMEOUT_MS
val output = client.getOutputStream() runCatching {
val request = readHttpRequest(input) val input = BufferedInputStream(client.getInputStream())
when { val output = client.getOutputStream()
request.method == "GET" && request.path == "/" -> respondHtml(output) val request = readHttpRequest(input)
request.method == "POST" && request.path == "/upload" -> handleUpload(request, output) when {
else -> respondText(output, 404, "Not found", "text/plain; charset=utf-8") request.method == "GET" && request.path == "/" -> respondHtml(output)
request.method == "POST" && request.path == "/upload" -> handleUpload(request, output)
else -> respondText(output, 404, "Not found", "text/plain; charset=utf-8")
}
}.onFailure {
Log.w(TAG, "Upload request failed", it)
} }
} }
} }
@@ -133,33 +140,78 @@ class UploadServer(
<pre id="result"></pre> <pre id="result"></pre>
<script> <script>
const result = document.getElementById('result'); const result = document.getElementById('result');
document.getElementById('uploadFolderBtn').onclick = async () => { const uploadButton = document.getElementById('uploadFolderBtn');
const IMAGE_REQUEST_TIMEOUT_MS = 90000;
const VIDEO_REQUEST_TIMEOUT_MS = 210000;
uploadButton.onclick = async () => {
result.textContent = '上传中...'; result.textContent = '上传中...';
const files = document.getElementById('folderFiles').files; const files = document.getElementById('folderFiles').files;
if (!files.length) {
result.textContent = '请选择文件后再上传。';
return;
}
let ok = 0; let ok = 0;
let fail = 0; let fail = 0;
const logs = []; const logs = [];
for (let i = 0; i < files.length; i++) { uploadButton.disabled = true;
const file = files[i]; try {
result.textContent = `上传中 ${'$'}{i + 1}/${'$'}{files.length}: ${'$'}{file.name}`; for (let i = 0; i < files.length; i++) {
const base64 = await toBase64(file); const file = files[i];
const resp = await fetch('/upload', { result.textContent = progressText('上传中', i + 1, files.length, file.name, ok, fail, logs);
method: 'POST', try {
headers: { 'Content-Type': 'application/json' }, const base64 = await toBase64(file);
body: JSON.stringify({ items: [{ fileName: file.name, base64, mimeType: file.type }] }) const resp = await fetchWithTimeout('/upload', {
}); method: 'POST',
const text = await resp.text(); headers: { 'Content-Type': 'application/json' },
logs.push(text); body: JSON.stringify({ items: [{ fileName: file.name, base64, mimeType: file.type }] })
try { }, uploadTimeoutMs(file));
const parsed = JSON.parse(text); const text = await resp.text();
ok += parsed.accepted || 0; logs.push(text);
fail += parsed.rejected || 0; try {
} catch (e) { const parsed = JSON.parse(text);
fail += 1; ok += parsed.accepted || 0;
fail += parsed.rejected || 0;
} catch (e) {
fail += 1;
}
} catch (e) {
fail += 1;
const reason = e.name === 'AbortError'
? '处理超时,已跳过该文件。'
: (e.message || String(e));
logs.push(JSON.stringify({
accepted: 0,
rejected: 1,
message: 'FAIL ' + file.name + ': ' + reason
}));
}
result.textContent = progressText('上传中', i + 1, files.length, file.name, ok, fail, logs);
} }
} finally {
uploadButton.disabled = false;
} }
result.textContent = `完成:成功 ${'$'}{ok},失败 ${'$'}{fail}\n\n` + logs.join('\n'); result.textContent = `完成:成功 ${'$'}{ok},失败 ${'$'}{fail}\n\n` + logs.join('\n');
}; };
function progressText(state, current, total, fileName, ok, fail, logs) {
const recent = logs.slice(-8).join('\n');
const summary = `${'$'}{state} ${'$'}{current}/${'$'}{total}: ${'$'}{fileName}\n成功 ${'$'}{ok},失败 ${'$'}{fail}`;
return recent ? summary + '\n\n最近结果\n' + recent : summary;
}
async function fetchWithTimeout(url, options, timeoutMs) {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeoutMs);
try {
return await fetch(url, { ...options, signal: controller.signal });
} finally {
clearTimeout(timer);
}
}
function uploadTimeoutMs(file) {
return isVideo(file) ? VIDEO_REQUEST_TIMEOUT_MS : IMAGE_REQUEST_TIMEOUT_MS;
}
function isVideo(file) {
return file.type.startsWith('video/') || /\.(mp4|avi|mov|wmv|flv|mkv|webm|3gp)$/i.test(file.name);
}
function toBase64(file) { function toBase64(file) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const reader = new FileReader(); const reader = new FileReader();
@@ -213,10 +265,17 @@ class UploadServer(
enrollment: SmartEnrollment, enrollment: SmartEnrollment,
report: (ok: Boolean, name: String, error: String?) -> Unit report: (ok: Boolean, name: String, error: String?) -> Unit
) { ) {
val startedAt = System.currentTimeMillis()
val timeoutMs = enrollmentTimeoutMs(mimeType, fileName)
Log.i(TAG, "Enrollment started: $fileName (${data.size} bytes)")
runCatching { runCatching {
val result = runBlocking { val result = runBlocking {
enrollment.enroll(data, mimeType, fileName) withTimeoutOrNull(timeoutMs) {
} enrollment.enroll(data, mimeType, fileName)
}
} ?: com.example.studentfaceregistry.face.EnrollmentResult.Failure(
"处理超时(超过 ${timeoutMs / 1000} 秒),已跳过该文件。"
)
when (result) { when (result) {
is com.example.studentfaceregistry.face.EnrollmentResult.Success -> { is com.example.studentfaceregistry.face.EnrollmentResult.Success -> {
@@ -240,8 +299,21 @@ class UploadServer(
} }
} }
}.onFailure { }.onFailure {
Log.w(TAG, "Enrollment failed: $fileName", it)
report(false, fileName, it.message) report(false, fileName, it.message)
} }
val elapsedMs = System.currentTimeMillis() - startedAt
Log.i(TAG, "Enrollment finished: $fileName in ${elapsedMs}ms")
}
private fun enrollmentTimeoutMs(mimeType: String, fileName: String): Long {
val lowerMimeType = mimeType.lowercase()
val lowerFileName = fileName.lowercase()
return if (lowerMimeType.startsWith("video/") || VIDEO_EXTENSIONS.any { lowerFileName.endsWith(it) }) {
VIDEO_ENROLLMENT_TIMEOUT_MS
} else {
IMAGE_ENROLLMENT_TIMEOUT_MS
}
} }
@Synchronized @Synchronized
@@ -329,6 +401,7 @@ class UploadServer(
private fun statusText(code: Int): String = when (code) { private fun statusText(code: Int): String = when (code) {
200 -> "OK" 200 -> "OK"
400 -> "Bad Request"
404 -> "Not Found" 404 -> "Not Found"
503 -> "Service Unavailable" 503 -> "Service Unavailable"
else -> "OK" else -> "OK"
@@ -343,6 +416,14 @@ class UploadServer(
} }
private fun jsonString(value: String): String = JSONObject.quote(value) private fun jsonString(value: String): String = JSONObject.quote(value)
companion object {
private const val TAG = "UploadServer"
private const val SOCKET_READ_TIMEOUT_MS = 120_000
private const val IMAGE_ENROLLMENT_TIMEOUT_MS = 45_000L
private const val VIDEO_ENROLLMENT_TIMEOUT_MS = 180_000L
private val VIDEO_EXTENSIONS = setOf(".mp4", ".avi", ".mov", ".wmv", ".flv", ".mkv", ".webm", ".3gp")
}
} }
private data class HttpRequest( private data class HttpRequest(