diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..e2909f8 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,65 @@ +# AGENTS.md + +Behavioral guidelines to reduce common LLM coding mistakes. Merge with project-specific instructions as needed. + +**Tradeoff:** These guidelines bias toward caution over speed. For trivial tasks, use judgment. + +## 1. Think Before Coding + +**Don't assume. Don't hide confusion. Surface tradeoffs.** + +Before implementing: +- State your assumptions explicitly. +- If multiple interpretations exist, present them - don't pick silently. +- If a simpler approach exists, say so. Push back when warranted. +- If something is unclear, stop. Name what's confusing. Ask. + +## 2. Simplicity First + +**Minimum code that solves the problem. Nothing speculative.** + +- No features beyond what was asked. +- No abstractions for single-use code. +- No "flexibility" or "configurability" that wasn't requested. +- No error handling for impossible scenarios. +- If you write 200 lines and it could be 50, rewrite it. + +Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify. + +## 3. Surgical Changes + +**Touch only what you must. Clean up only your own mess.** + +When editing existing code: +- Don't "improve" adjacent code, comments, or formatting. +- Don't refactor things that aren't broken. +- Match existing style, even if you'd do it differently. +- If you notice unrelated dead code, mention it - don't delete it. + +When your changes create orphans: +- Remove imports/variables/functions that YOUR changes made unused. +- Don't remove pre-existing dead code unless asked. + +The test: Every changed line should trace directly to the user's request. + +## 4. Goal-Driven Execution + +**Define success criteria. Loop until verified.** + +Transform tasks into verifiable goals: +- "Add validation" → "Write tests for invalid inputs, then make them pass" +- "Fix the bug" → "Write a test that reproduces it, then make it pass" +- "Refactor X" → "Ensure tests pass before and after" + +For multi-step tasks, state a brief plan: +``` +1. [Step] → verify: [check] +2. [Step] → verify: [check] +3. [Step] → verify: [check] +``` + +Strong success criteria let you loop independently. Weak criteria ("make it work") require constant clarification. + +--- + +**These guidelines are working if:** fewer unnecessary changes in diffs, fewer rewrites due to overcomplication, and clarifying questions come before implementation rather than after mistakes. diff --git a/README.md b/README.md index baba205..38e8486 100644 --- a/README.md +++ b/README.md @@ -28,13 +28,12 @@ http://192.168.1.23:8080 让电脑和手机连接同一个 Wi-Fi，然后在电脑浏览器打开这个地址。网页里可以一次选择多张学生头像，点击上传后，手机会本地检测人脸、生成 embedding 并写入本地数据库。 -网页同时支持两种批量导入方式： +网页支持按文件夹批量导入： -- 选择一个文件夹，自动上传文件夹里的所有图片 -- 上传一个 `.zip` 压缩包，手机端自动解压后入库 +- 选择一个文件夹，自动上传文件夹里的图片或视频 文件夹选择在 Chrome / Edge 这类支持目录选择的浏览器里效果最好。 -大批量导入时优先选择文件夹上传；网页会逐张发送图片，手机端内存压力更小。 +网页会逐个发送文件，手机端内存压力更小。 ## 头像命名规则 diff --git a/app/src/main/java/com/example/studentfaceregistry/face/SmartEnrollment.kt b/app/src/main/java/com/example/studentfaceregistry/face/SmartEnrollment.kt new file mode 100644 index 0000000..d7dea12 --- /dev/null +++ b/app/src/main/java/com/example/studentfaceregistry/face/SmartEnrollment.kt @@ -0,0 +1,484 @@ +package com.example.studentfaceregistry.face + +import android.content.Context +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.graphics.Rect +import android.media.MediaDataSource +import android.media.MediaMetadataRetriever +import com.google.mlkit.vision.common.InputImage +import com.google.mlkit.vision.face.Face +import com.google.mlkit.vision.face.FaceDetection +import com.google.mlkit.vision.face.FaceDetectorOptions +import kotlinx.coroutines.tasks.await +import kotlin.math.atan2 +import kotlin.math.sqrt + +/** + * 智能注册器 + * 支持图片和视频两种输入方式，自动选择最大人脸进行注册 + * + * 功能特性： + * 1. 自动识别输入类型（图片或视频） + * 2. 自动检测并选择最大的人脸 + * 3. 视频注册时自动提取多角度帧并融合特征 + */ +class SmartEnrollment(context: Context) { + + // 高精度人脸检测器（用于注册） + private val detector = FaceDetection.getClient( + FaceDetectorOptions.Builder() + .setPerformanceMode(FaceDetectorOptions.PERFORMANCE_MODE_ACCURATE) + .setLandmarkMode(FaceDetectorOptions.LANDMARK_MODE_ALL) + .setMinFaceSize(0.03f) + .build() + ) + + private val embedder = FaceEmbedder(context, FaceEmbedder.ModelType.ARCFACE) + private val aligner = FaceAligner() + private val preprocessor = ImagePreprocessor() + + /** + * 智能注册入口 + * @param data 数据字节（图片或视频） + * @param mimeType MIME 类型，如 "image/jpeg", "video/mp4" + * @param fileName 文件名（用于提取学号和姓名） + * @return 注册结果 + */ + suspend fun enroll( + data: ByteArray, + mimeType: String?, + fileName: String + ): EnrollmentResult { + return if (isVideo(mimeType, fileName)) { + enrollFromVideo(data) + } else { + enrollFromImage(data) + } + } + + /** + * 从图片注册 + */ + private suspend fun enrollFromImage(data: ByteArray): EnrollmentResult { + return try { + // 解码图片 + val bitmap = decodeBitmap(data) + + // 检测人脸并只保留可安全裁剪的人脸区域 + val faces = detectUsableFaces(bitmap) + + if (faces.isEmpty()) { + return EnrollmentResult.Failure("未检测到人脸，请换更清晰或更正面的照片。") + } + + // 选择最大的人脸 + val largestFace = faces.maxBy { faceInfo -> + faceInfo.faceArea + } + + // 裁剪人脸 + val croppedFace = cropFace(bitmap, largestFace.cropRect) + + // 对齐 + val alignedFace = alignFace(croppedFace, largestFace.face, largestFace.cropRect) + + // 预处理 + val preprocessedFace = preprocessor.preprocess(alignedFace) + + // 提取特征 + val embedding = embedder.embed(preprocessedFace) + + EnrollmentResult.Success(embedding, EnrollmentSourceType.IMAGE) + } catch (e: Exception) { + EnrollmentResult.Failure("图片处理失败：${e.message}") + } + } + + /** + * 从视频注册 + * @param data 视频字节 + * @param minFrames 最少帧数 + * @param maxFrames 最大帧数（用于融合） + */ + private suspend fun enrollFromVideo( + data: ByteArray, + minFrames: Int = 10, + maxFrames: Int = 30 + ): EnrollmentResult { + return try { + // 提取视频帧 + val frames = extractVideoFrames(data, maxFrames * 2) + + if (frames.size < minFrames) { + return EnrollmentResult.Failure( + "视频有效帧数不足（检测到 ${frames.size} 帧，需要至少 $minFrames 帧），" + + "请确保视频中人脸清晰且持续展示足够时间。" + ) + } + + // 对每帧选择最大人脸 + val faceFrames = frames.mapNotNull { bitmap -> + val faces = detectUsableFaces(bitmap) + if (faces.isEmpty()) return@mapNotNull null + + val largestFace = faces.maxBy { faceInfo -> + faceInfo.faceArea + } + + FaceFrameInfo(bitmap, largestFace.face, largestFace.cropRect, largestFace.faceArea) + } + + if (faceFrames.size < minFrames) { + return EnrollmentResult.Failure( + "视频有效人脸帧数不足（检测到 ${faceFrames.size} 帧，需要至少 $minFrames 帧）" + ) + } + + // 计算每帧的角度并分组 + val groupedFrames = groupFramesByAngle(faceFrames) + + // 从每组选择质量最好的帧 + val selectedFrames = selectBestFramesFromGroups(groupedFrames, maxFrames) + + // 提取每帧的特征 + val embeddings = selectedFrames.map { frameInfo -> + val cropped = cropFace(frameInfo.bitmap, frameInfo.cropRect) + val aligned = alignFace(cropped, frameInfo.face, frameInfo.cropRect) + val preprocessed = preprocessor.preprocess(aligned) + embedder.embed(preprocessed) + } + + // 融合特征 + val fusedEmbedding = fuseEmbeddings(embeddings) + + // 计算角度覆盖 + val yawRange = calculateAngleRange(selectedFrames, { f -> yawAngle(f.face) }) + val pitchRange = calculateAngleRange(selectedFrames, { f -> pitchAngle(f.face) }) + + EnrollmentResult.Success( + embedding = fusedEmbedding, + sourceType = EnrollmentSourceType.VIDEO, + frameCount = selectedFrames.size, + yawRange = yawRange, + pitchRange = pitchRange + ) + } catch (e: Exception) { + EnrollmentResult.Failure("视频处理失败：${e.message}") + } + } + + /** + * 提取视频帧 + */ + private fun extractVideoFrames(videoData: ByteArray, maxFrames: Int): List { + val retriever = MediaMetadataRetriever() + val dataSource = ByteArrayVideoDataSource(videoData) + + return try { + retriever.setDataSource(dataSource) + + val duration = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION)?.toLongOrNull() ?: 0 + val frameInterval = maxOf(100, (duration / maxFrames).toInt()) // 至少每 100ms 一帧 + + val frames = mutableListOf() + var timeUs = 0L + + while (timeUs < duration * 1000) { + try { + val bitmap = retriever.getFrameAtTime(timeUs, MediaMetadataRetriever.OPTION_CLOSEST) + if (bitmap != null && !bitmap.isRecycled) { + frames.add(bitmap) + } + } catch (e: Exception) { + // 跳过错误帧 + } + timeUs += (frameInterval * 1000).toLong() // 转换为微秒 + } + + frames + } finally { + retriever.release() + dataSource.close() + } + } + + /** + * 计算偏航角（左右转头） + */ + private fun yawAngle(face: Face): Float { + val leftCheek = face.leftCheek ?: return 0f + val rightCheek = face.rightCheek ?: return 0f + if (!leftCheek.visible || !rightCheek.visible) return 0f + + val dx = rightCheek.position.x - leftCheek.position.x + val dy = rightCheek.position.y - leftCheek.position.y + return atan2(dy.toDouble(), dx.toDouble()).toFloat() * 180f / 3.14159 + } + + /** + * 计算俯仰角（上下点头） + */ + private fun pitchAngle(face: Face): Float { + val nose = face.nose ?: return 0f + val leftEye = face.leftEye ?: return 0f + val rightEye = face.rightEye ?: return 0f + if (!nose.visible || !leftEye.visible || !rightEye.visible) return 0f + + val eyeCenterY = (leftEye.position.y + rightEye.position.y) / 2f + val dx = nose.position.x - (leftEye.position.x + rightEye.position.x) / 2f + val dy = nose.position.y - eyeCenterY + return atan2(dy.toDouble(), dx.toDouble()).toFloat() * 180f / 3.14159 + } + + /** + * 按角度分组 + */ + private fun groupFramesByAngle(frames: List): Map> { + val buckets = mutableMapOf>() + + for (frame in frames) { + val yaw = yawAngle(frame.face) + val pitch = pitchAngle(frame.face) + val bucket = AngleBucket( + yawBucket = when { + yaw < -30 -> -2 + yaw < -10 -> -1 + yaw < 10 -> 0 + yaw < 30 -> 1 + else -> 2 + }, + pitchBucket = when { + pitch < -15 -> -1 + pitch < 15 -> 0 + else -> 1 + } + ) + if (!buckets.containsKey(bucket)) buckets[bucket] = mutableListOf() + buckets[bucket]!!.add(frame) + } + + return buckets + } + + /** + * 从每组选择最佳帧 + */ + private fun selectBestFramesFromGroups( + groups: Map>, + maxFrames: Int + ): List { + val selected = mutableListOf() + + // 按组内最大人脸面积排序 + val sortedGroups = groups.entries.sortedByDescending { entry -> + entry.value.maxOfOrNull { + it.faceArea + } ?: 0 + } + + for ((_, frames) in sortedGroups) { + val sortedFrames = frames.sortedByDescending { + it.faceArea + } + for (frame in sortedFrames) { + if (selected.size >= maxFrames) break + selected.add(frame) + } + } + + return selected + } + + /** + * 融合多个特征向量 + */ + private fun fuseEmbeddings(embeddings: List): FloatArray { + if (embeddings.isEmpty()) throw IllegalArgumentException("embeddings is empty") + if (embeddings.size == 1) return embeddings[0] + + val size = embeddings[0].size + val sum = FloatArray(size) + + for (embedding in embeddings) { + for (i in 0 until size) { + sum[i] += embedding[i] + } + } + + // 平均 + for (i in 0 until size) { + sum[i] /= embeddings.size + } + + // L2 归一化 + return l2Normalize(sum) + } + + /** + * 计算角度范围 + */ + private fun calculateAngleRange( + frames: List, + angleExtractor: (FaceFrameInfo) -> Float + ): Float { + if (frames.isEmpty()) return 0f + val angles = frames.map(angleExtractor) + return angles.maxOrNull()!! - angles.minOrNull()!! + } + + /** + * 裁剪人脸 + */ + private fun cropFace(bitmap: Bitmap, box: Rect): Bitmap { + return Bitmap.createBitmap(bitmap, box.left, box.top, box.width(), box.height()) + } + + private suspend fun detectUsableFaces(bitmap: Bitmap): List { + return detector.process(InputImage.fromBitmap(bitmap, 0)).await() + .mapNotNull { face -> + val cropRect = safeCropRect(bitmap, face.boundingBox) ?: return@mapNotNull null + val faceArea = face.boundingBox.width() * face.boundingBox.height() + DetectedFace(face, cropRect, faceArea) + } + } + + private fun safeCropRect(bitmap: Bitmap, box: Rect): Rect? { + if (box.width() <= 0 || box.height() <= 0) return null + + val padding = (maxOf(box.width(), box.height()) * 0.25f).toInt() + val left = (box.left - padding).coerceIn(0, bitmap.width) + val top = (box.top - padding).coerceIn(0, bitmap.height) + val right = (box.right + padding).coerceIn(0, bitmap.width) + val bottom = (box.bottom + padding).coerceIn(0, bitmap.height) + if (right - left <= 1 || bottom - top <= 1) return null + + return Rect(left, top, right, bottom) + } + + /** + * 对齐人脸 + */ + private fun alignFace(bitmap: Bitmap, face: Face, cropRect: Rect): Bitmap { + val leftEye = face.leftEye ?: return bitmap + val rightEye = face.rightEye ?: return bitmap + if (!leftEye.visible || !rightEye.visible) return bitmap + + return aligner.align( + bitmap = bitmap, + leftEyeX = leftEye.position.x, + leftEyeY = leftEye.position.y, + rightEyeX = rightEye.position.x, + rightEyeY = rightEye.position.y, + faceLeft = cropRect.left, + faceTop = cropRect.top + ) + } + + /** + * 解码 Bitmap + */ + private fun decodeBitmap(data: ByteArray): Bitmap { + val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true } + BitmapFactory.decodeByteArray(data, 0, data.size, bounds) + + val maxSide = maxOf(bounds.outWidth, bounds.outHeight).coerceAtLeast(1) + var sampleSize = 1 + while (maxSide / sampleSize > 1280) sampleSize *= 2 + + val options = BitmapFactory.Options().apply { + inSampleSize = sampleSize + inPreferredConfig = Bitmap.Config.ARGB_8888 + } + return BitmapFactory.decodeByteArray(data, 0, data.size, options) + ?: throw IllegalArgumentException("Unable to decode image") + } + + /** + * L2 归一化 + */ + private fun l2Normalize(values: FloatArray): FloatArray { + var sum = 0f + for (value in values) sum += value * value + val norm = sqrt(sum.coerceAtLeast(1e-12f)) + return FloatArray(values.size) { values[it] / norm } + } + + /** + * 判断是否为视频 + */ + private fun isVideo(mimeType: String?, fileName: String): Boolean { + val lowerMimeType = mimeType?.lowercase().orEmpty() + val lowerFileName = fileName.lowercase() + return lowerMimeType.startsWith("video/") || + lowerMimeType in VIDEO_MIME_TYPES || + VIDEO_EXTENSIONS.any { lowerFileName.endsWith(it) } + } + + override fun close() { + detector.close() + embedder.close() + } + + companion object { + private val VIDEO_EXTENSIONS = setOf(".mp4", ".avi", ".mov", ".wmv", ".flv", ".mkv", ".webm", ".3gp") + private val VIDEO_MIME_TYPES = setOf("application/mp4") + } +} + +private data class DetectedFace( + val face: Face, + val cropRect: Rect, + val faceArea: Int +) + +/** + * 人脸帧信息 + */ +private data class FaceFrameInfo( + val bitmap: Bitmap, + val face: Face, + val cropRect: Rect, + val faceArea: Int +) + +private class ByteArrayVideoDataSource(private val data: ByteArray) : MediaDataSource() { + override fun readAt(position: Long, buffer: ByteArray, offset: Int, size: Int): Int { + if (position >= data.size) return -1 + val length = minOf(size, data.size - position.toInt()) + System.arraycopy(data, position.toInt(), buffer, offset, length) + return length + } + + override fun getSize(): Long = data.size.toLong() + + override fun close() = Unit +} + +/** + * 角度分组 + */ +private data class AngleBucket(val yawBucket: Int, val pitchBucket: Int) + +/** + * 注册结果 + */ +sealed class EnrollmentResult { + data class Success( + val embedding: FloatArray, + val sourceType: EnrollmentSourceType = EnrollmentSourceType.IMAGE, + val frameCount: Int = 1, + val yawRange: Float = 0f, + val pitchRange: Float = 0f + ) : EnrollmentResult() + + data class Failure(val reason: String) : EnrollmentResult() +} + +/** + * 注册源类型 + */ +enum class EnrollmentSourceType { + IMAGE, // 图片注册 + VIDEO // 视频注册 +} diff --git a/app/src/main/java/com/example/studentfaceregistry/upload/UploadServer.kt b/app/src/main/java/com/example/studentfaceregistry/upload/UploadServer.kt index 1d47f19..cc00500 100644 --- a/app/src/main/java/com/example/studentfaceregistry/upload/UploadServer.kt +++ b/app/src/main/java/com/example/studentfaceregistry/upload/UploadServer.kt @@ -1,15 +1,14 @@ package com.example.studentfaceregistry.upload import android.content.Context -import android.graphics.BitmapFactory import android.util.Base64 import com.example.studentfaceregistry.data.Student import com.example.studentfaceregistry.data.StudentRepository import com.example.studentfaceregistry.face.FaceProcessor +import com.example.studentfaceregistry.face.SmartEnrollment import kotlinx.coroutines.runBlocking import org.json.JSONObject import java.io.BufferedInputStream -import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream import java.io.OutputStream import java.net.Inet4Address @@ -17,7 +16,6 @@ import java.net.NetworkInterface import java.net.ServerSocket import java.net.Socket import java.nio.charset.StandardCharsets -import java.util.zip.ZipInputStream import kotlin.concurrent.thread class UploadServer( @@ -27,10 +25,12 @@ class UploadServer( ) : AutoCloseable { private var serverSocket: ServerSocket? = null @Volatile private var running = false + private lateinit var smartEnrollment: SmartEnrollment fun start() { if (running) return running = true + smartEnrollment = SmartEnrollment(context) serverSocket = ServerSocket(8080) thread(name = "upload-server", isDaemon = true) { while (running) { @@ -52,6 +52,7 @@ class UploadServer( override fun close() { running = false runCatching { serverSocket?.close() } + runCatching { smartEnrollment.close() } } private fun handle(socket: Socket) { @@ -70,8 +71,7 @@ class UploadServer( } private fun handleUpload(request: HttpRequest, output: OutputStream) { - val processor = processorProvider() - if (processor == null) { + if (processorProvider() == null) { respondText(output, 503, "Missing facenet.tflite model", "text/plain; charset=utf-8") return } @@ -82,10 +82,10 @@ class UploadServer( runCatching { val payload = JSONObject(bodyText) - val zipBase64 = payload.optString("zipBase64", "") - if (zipBase64.isNotBlank()) { - val zipBytes = Base64.decode(zipBase64, Base64.DEFAULT) - processZip(zipBytes, processor) { ok, name, error -> + // 单文件或文件列表上传（支持图片和视频） + val items = parseUploadItems(payload) + items.forEachIndexed { index, item -> + processUploadItem(item.fileName.ifBlank { "student_$index" }, item.base64, item.mimeType) { ok, name, error -> if (ok) { accepted += 1 notices += "OK $name" @@ -94,19 +94,6 @@ class UploadServer( notices += "FAIL $name: $error" } } - } else { - val images = parseImages(payload) - images.forEachIndexed { index, item -> - processImage(item.fileName.ifBlank { "student_$index.jpg" }, item.base64, processor) { ok, name, error -> - if (ok) { - accepted += 1 - notices += "OK $name" - } else { - rejected += 1 - notices += "FAIL $name: $error" - } - } - } } }.onFailure { respondText(output, 400, """{"accepted":0,"rejected":0,"message":${jsonString(it.message ?: "invalid request")}}""", "application/json; charset=utf-8") @@ -126,7 +113,7 @@ class UploadServer( - 学生头像上传 + 学生人脸注册上传 -

学生头像批量上传

-

在电脑上选择头像，点击上传后，手机会本地生成人脸特征并保存。文件名建议使用 学号_姓名.jpg。

+

学生人脸注册上传

+

选择学生图片或视频后上传，手机会自动按文件类型注册，并优先使用画面中最大的人脸。文件名建议使用 学号_姓名.jpg 或 学号_姓名.mp4。


-
+
-
-
-
- -