Merge pull request #24 from dkwstas/vision_integration

Vision integration
2026-06-01 02:25:54 +03:00
parent 2fb15443ef 98bffc4117
commit a7974c78f3
13 changed files with 386 additions and 30 deletions
@@ -2,4 +2,6 @@ node_modules
 # Keep environment variables out of version control
 .env

-**/generated/prisma
+/generated/prisma
+
+/generated/prisma
@@ -0,0 +1,7 @@
+{
+    "semi": true,
+    "singleQuote": false,
+    "tabWidth": 4,
+    "trailingComma": "es5",
+    "printWidth": 100
+}
@@ -23,6 +23,7 @@
        "@types/express": "^5.0.6",
        "@types/express-session": "^1.19.0",
        "@types/node": "^25.7.0",
+        "prettier": "^3.8.3",
        "tsx": "^4.21.0",
        "typescript": "^6.0.3"
      }
@@ -6653,6 +6654,22 @@
      "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==",
      "license": "MIT"
    },
+    "node_modules/prettier": {
+      "version": "3.8.3",
+      "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.8.3.tgz",
+      "integrity": "sha512-7igPTM53cGHMW8xWuVTydi2KO233VFiTNyF5hLJqpilHfmn8C8gPf+PS7dUT64YcXFbiMGZxS9pCSxL/Dxm/Jw==",
+      "dev": true,
+      "license": "MIT",
+      "bin": {
+        "prettier": "bin/prettier.cjs"
+      },
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "url": "https://github.com/prettier/prettier?sponsor=1"
+      }
+    },
    "node_modules/prisma": {
      "version": "6.19.3",
      "resolved": "https://registry.npmjs.org/prisma/-/prisma-6.19.3.tgz",
@@ -9,12 +9,15 @@
  "scripts": {
    "dev": "tsx src/index.ts",
    "build": "tsc",
-    "start": "node dist/index.js"
+    "start": "node dist/index.js",
+    "format": "prettier --write \"src/**/*.{ts,tsx}\"",
+    "format:check": "prettier --check \"src/**/*.{ts,tsx}\""
  },
  "devDependencies": {
    "@types/express": "^5.0.6",
    "@types/express-session": "^1.19.0",
    "@types/node": "^25.7.0",
+    "prettier": "^3.8.3",
    "tsx": "^4.21.0",
    "typescript": "^6.0.3"
  },
@@ -1,14 +0,0 @@
-// This file was generated by Prisma, and assumes you have installed the following:
-// npm install --save-dev prisma dotenv
-import "dotenv/config";
-import { defineConfig } from "prisma/config";
-
-export default defineConfig({
-  schema: "prisma/schema.prisma",
-  migrations: {
-    path: "prisma/migrations",
-  },
-  datasource: {
-    url: process.env["DATABASE_URL"],
-  },
-});
@@ -8,4 +8,6 @@ export const componentLoader = new ComponentLoader()

 export const Components = {
    Dashboard: componentLoader.add('Dashboard', path.join(__dirname, './components/Dashboard.tsx'), 'components'),
+    UploadFace: componentLoader.add('UploadFace', path.join(__dirname, './components/UploadFace.tsx'), 'components'),
+    FaceEmbeddingField: componentLoader.add('FaceEmbeddingField', path.join(__dirname, './components/FaceEmbeddingField.tsx'), 'components'),
 }
@@ -0,0 +1,26 @@
+import React from 'react'
+import { BasePropertyProps } from 'adminjs'
+import { Label } from '@adminjs/design-system'
+
+const FaceEmbeddingField: React.FC<BasePropertyProps> = ({ record, property }) => {
+    const has = !!record?.params?.faceEmbedding
+    return (
+        <div style={{ marginBottom: '16px' }}>
+            <Label>{property.label}</Label>
+            <div style={{ marginTop: 6 }}>
+                <span style={{
+                    fontSize: '12px',
+                    fontWeight: 600,
+                    padding: '3px 10px',
+                    borderRadius: '4px',
+                    background: has ? '#dcfce7' : '#fee2e2',
+                    color: has ? '#16a34a' : '#dc2626',
+                }}>
+                    {has ? 'SET' : 'UNSET'}
+                </span>
+            </div>
+        </div>
+    )
+}
+
+export default FaceEmbeddingField
@@ -0,0 +1,96 @@
+import React, { useState, useRef } from 'react'
+import { Box, H3, Text, Button, MessageBox } from '@adminjs/design-system'
+import { ActionProps, useRecord } from 'adminjs'
+
+const UploadFace: React.FC<ActionProps> = ({ record, action }) => {
+    const [preview, setPreview] = useState<string | null>(null)
+    const [base64, setBase64] = useState<string | null>(null)
+    const [loading, setLoading] = useState(false)
+    const [notice, setNotice] = useState<{ message: string, type: 'success' | 'error' } | null>(null)
+    const inputRef = useRef<HTMLInputElement>(null)
+
+    const handleFile = (e: React.ChangeEvent<HTMLInputElement>) => {
+        const file = e.target.files?.[0]
+        if (!file) return
+        const reader = new FileReader()
+        reader.onload = () => {
+            const result = reader.result as string
+            setPreview(result)
+            setBase64(result.split(',')[1]) // strip data:image/jpeg;base64,
+        }
+        reader.readAsDataURL(file)
+    }
+
+    const handleSubmit = async () => {
+        if (!base64) return
+        setLoading(true)
+        setNotice(null)
+        try {
+            const res = await fetch(
+                `/admin/api/resources/User/records/${record?.params.id}/uploadFace`,
+                {
+                    method: 'POST',
+                    headers: { 'Content-Type': 'application/json' },
+                    credentials: 'include',
+                    body: JSON.stringify({ base64 }),
+                }
+            )
+            const data = await res.json()
+            setNotice(data.notice)
+        } catch (e: any) {
+            setNotice({ message: e.message, type: 'error' })
+        } finally {
+            setLoading(false)
+        }
+    }
+
+    return (
+        <Box padding="xl">
+            <H3 marginBottom="md">Upload face image</H3>
+            <Text color="grey60" marginBottom="lg">
+                Upload a JPEG photo of the user to generate and store their face embedding.
+            </Text>
+
+            {notice && (
+                <MessageBox
+                    message={notice.message}
+                    variant={notice.type === 'success' ? 'success' : 'danger'}
+                    marginBottom="lg"
+                />
+            )}
+
+            <Box marginBottom="lg">
+                <input
+                    ref={inputRef}
+                    type="file"
+                    accept="image/jpeg"
+                    onChange={handleFile}
+                    style={{ display: 'none' }}
+                />
+                <Button onClick={() => inputRef.current?.click()} variant="outlined">
+                    Choose JPEG
+                </Button>
+            </Box>
+
+            {preview && (
+                <Box marginBottom="lg">
+                    <img
+                        src={preview}
+                        alt="Preview"
+                        style={{ width: 160, height: 160, objectFit: 'cover', borderRadius: 8, border: '1px solid #e5e7eb' }}
+                    />
+                </Box>
+            )}
+
+            <Button
+                onClick={handleSubmit}
+                disabled={!base64 || loading}
+                variant="contained"
+            >
+                {loading ? 'Processing…' : 'Save embedding'}
+            </Button>
+        </Box>
+    )
+}
+
+export default UploadFace
@@ -1,5 +1,49 @@
 import { prisma } from "../../../lib/prisma.js";
 import { getModelByName } from "@adminjs/prisma";
+import { spawn } from "child_process";
+import { Components } from '../components.js'
+import { fileURLToPath } from 'url'
+import path from 'path'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+const execPath = '../../../../bin';
+const binaryPath = path.join(__dirname, execPath, 'extractEmbeddings')
+
+async function extractEmbedding(base64: string): Promise<string> {
+    return new Promise((resolve, reject) => {
+        const proc = spawn(binaryPath, [], {
+            cwd: path.join(__dirname, execPath)
+        })
+        let stdout = ''
+        let stderr = ''
+
+        proc.stdout.on('data', (d) => stdout += d)
+        proc.stderr.on('data', (d) => stderr += d)
+
+        proc.stdin.on('error', (err) => { });
+
+        proc.on('error', (err) => {
+            reject(new Error(`Failed to start binary: ${err.message}`))
+        })
+
+        proc.on('close', (code) => {
+            console.log('extractEmbeddings exited with code', code)
+            console.log('stdout:', stdout)
+            console.log('stderr:', stderr)
+            if (code !== 0) return reject(new Error(stderr || `exited with code ${code}`))
+            try {
+                const result = JSON.parse(stdout)
+                if (!result.success) return reject(new Error(result.error))
+                resolve(JSON.stringify(result.descriptor))
+            } catch (e) {
+                reject(new Error(`Failed to parse output: ${stdout}`))
+            }
+        })
+
+        proc.stdin.write(base64)
+        proc.stdin.end()
+    })
+}

 export default {
    resource: {
@@ -10,14 +54,56 @@ export default {
        titleProperty: "id",
        navigation: { icon: "User" },
        listProperties: ["id", "firstName", "lastName"],
-        showProperties: ["id", "firstName", "lastName"],
+        showProperties: ["id", "firstName", "lastName", "faceEmbedding"],
        editProperties: ["firstName", "lastName"],
        filterProperties: ["firstName", "lastName"],
        properties: {
            faceEmbedding: {
-                isVisible: false,
-                type: "string"
+                isVisible: { list: false, show: true, edit: false, filter: false },
+                type: "string",
+                components: {
+                    list: Components.FaceEmbeddingField,
+                    show: Components.FaceEmbeddingField,
+                },
            },
-        }
+        },
+        actions: {
+            uploadFace: {
+                actionType: 'record',
+                icon: 'Camera',
+                label: 'Upload face',
+                showInDrawer: true,
+                handler: async (request: any, response: any, context: any) => {
+                    const { record, currentAdmin } = context
+                    if (request.method === 'post') {
+                        const { base64 } = request.payload
+                        if (!base64) {
+                            return {
+                                record: record.toJSON(currentAdmin),
+                                notice: { message: 'No image provided', type: 'error' },
+                            }
+                        }
+                        try {
+                            const embedding = await extractEmbedding(base64)
+                            await prisma.user.update({
+                                where: { id: record.params.id },
+                                data: { faceEmbedding: embedding },
+                            })
+                            return {
+                                record: record.toJSON(currentAdmin),
+                                notice: { message: 'Face embedding saved!', type: 'success' },
+                            }
+                        } catch (e: any) {
+                            return {
+                                record: record.toJSON(currentAdmin),
+                                notice: { message: `Failed: ${e.message}`, type: 'error' },
+                            }
+                        }
+                    }
+                    return { record: record.toJSON(currentAdmin) }
+                },
+                component: Components.UploadFace,
+            },
+        },
    },
 }
@@ -23,6 +23,7 @@ interface Config {
        transitionCleanupInterval: number;
        lossThreshold: number;
        userTTL: number;
+        verifyTimeout: number;
    };
 }

@@ -50,7 +51,8 @@ const config: Config = {
        transitionTTL: 5 * 60 * 1000,
        transitionCleanupInterval: 60 * 1000,
        lossThreshold: 5000,
-        userTTL: 3 * 60 * 1000
+        userTTL: 3 * 60 * 1000,
+        verifyTimeout: 30 * 1000
    },
 }

@@ -38,7 +38,7 @@ export async function updateRoomOccupancyListener(message: string, channel: stri
    const redis = getRedis();

    if (!lastRoomID) {
-        console.log("[!] Key not found in local mapι")
+        console.log("[!] Key not found in local map")
    } else {
        await redis.decr(`room:${lastRoomID}`);
    }
@@ -103,8 +103,6 @@ export async function analyzeData(roomID: string, metrics: {
            const user = newUser();
            console.log(`[Redis] Created USER: ${user.psuedoName} USER_ID: ${user.pseudoID}`);

-            // trigger camera
-
            await redis.set(`user:${userID}`, JSON.stringify({
                userID: user.pseudoID,
                name: user.psuedoName,
@@ -141,8 +139,6 @@ export async function analyzeData(roomID: string, metrics: {

        let transition = transitions.get(userID);

-        // update room occupancy
-
        if (!transition) {
            console.log("[!] Starting new transition");
            transition = new RoomTransition();
@@ -150,8 +146,6 @@ export async function analyzeData(roomID: string, metrics: {
        }

        if (transition.shouldTransitionTo(roomID, metrics.rssi, resObj["rssi"], resObj["timestamp"])) {
-            // trigger camera
-
            console.log(`[!] Transition done ${resObj["room"]} -> ${roomID}`);

            await redis.decr(`room:${resObj["room"]}`);
@@ -162,7 +156,7 @@ export async function analyzeData(roomID: string, metrics: {
            resObj["rssi"] = metrics.rssi;
            resObj["room"] = roomID;
            resObj["timestamp"] = Date.now()
-
+            resObj["verified"] = false;

            await redis.set(`user:${userID}`, JSON.stringify(resObj), { PX: config.core.userTTL });
        } else
@@ -1,5 +1,6 @@
 import { getRoomIDs } from "../api/livedata/livedata.repository.js";
 import { initRedis, initSubRedis } from "../redis/redis.js";
+import { initCamera } from "./camera.js";
 import * as mqtt from "./mqtt.js";
 import { cleanupWorker } from "./transition.js";

@@ -19,6 +20,8 @@ export async function bootstrap() {

        cleanupWorker();

+        await initCamera();
+
        console.log("[!] Core started.");
    } catch (err: any) {
        console.log("[!] Bootstrap failed:", err.message);
@@ -0,0 +1,132 @@
+import { fileURLToPath } from 'url'
+import path from 'path'
+import { spawn } from "child_process";
+import { getRedis } from '../redis/redis.js';
+import { prisma } from '../lib/prisma.js';
+import config from '../config/config.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+const execPath = '../../bin';
+const binaryPath = path.join(__dirname, execPath, 'cameraSession')
+
+type InputPacket = {
+    uuid: string
+    descriptor: number[]
+}
+
+type ResultPacket = {
+    uuid: string
+    verified: boolean
+}
+
+export async function getUnverifiedUserKeys(): Promise<string[]> {
+    const redis = getRedis()
+
+    const keys = await redis.keys('user:*')
+
+    const unverified: string[] = []
+
+    await Promise.all(keys.map(async (key) => {
+        const raw = await redis.get(key)
+        if (!raw) return
+        const session = JSON.parse(raw)
+        if (!session.verified) unverified.push(key.replace('user:', ''))
+    }))
+
+    return unverified
+}
+
+export async function getUnverifiedUsersWithEmbeddings(): Promise<{ uuid: string, descriptor: number[] }[]> {
+    const userIds = await getUnverifiedUserKeys()
+
+    const users = await prisma.user.findMany({
+        where: {
+            id: { in: userIds },
+            faceEmbedding: { not: null },
+        },
+        select: { id: true, faceEmbedding: true },
+    })
+
+    return users.map((user) => ({
+        uuid: user.id,
+        descriptor: JSON.parse(user.faceEmbedding!),
+    }))
+}
+
+async function cameraSession(packets: InputPacket[]): Promise<ResultPacket[] | null> {
+    return new Promise((resolve, reject) => {
+        const proc = spawn(binaryPath, [], {
+            cwd: path.join(__dirname, execPath)
+        })
+        let stdout = ''
+        let stderr = ''
+
+        proc.stdout.on('data', (d) => stdout += d)
+        proc.stderr.on('data', (d) => stderr += d)
+        proc.stdin.on('error', () => { })
+
+        proc.on('error', (err) => {
+            reject(new Error(`Failed to start binary: ${err.message}`))
+        })
+
+        proc.on('close', (code) => {
+            console.log('cameraSession exited with code', code)
+            const trimmed = stdout.trim()
+            if (trimmed === 'no face found.') return resolve(null)
+            if (code !== 0) return reject(new Error(stderr || `exited with code ${code}`))
+            try {
+                const results: ResultPacket[] = JSON.parse(trimmed)
+                resolve(results)
+            } catch (e) {
+                reject(new Error(`Failed to parse output: ${trimmed}`))
+            }
+        })
+
+        for (const packet of packets) {
+            proc.stdin.write(JSON.stringify(packet) + '\n')
+        }
+        proc.stdin.end()
+    })
+}
+
+const setUnverified = async (uuid: string) => {
+    const redis = getRedis()
+    const key = `user:${uuid}`
+    const raw = await redis.get(key)
+    if (!raw) return
+    const session = JSON.parse(raw)
+    session.verified = false
+    await redis.set(key, JSON.stringify(session))
+    console.log(`[Camera] Unverified user ${uuid} due to timeout`)
+}
+
+const run = async () => {
+    const redis = getRedis();
+
+    try {
+        const packets = await getUnverifiedUsersWithEmbeddings()
+        if (packets.length > 0) {
+            const results = await cameraSession(packets)
+            if (results) {
+                await Promise.all(results.map(async (r) => {
+                    if (!r.verified) return
+                    const key = `user:${r.uuid}`
+                    const raw = await redis.get(key)
+                    if (!raw) return
+                    const session = JSON.parse(raw)
+                    session.verified = true
+                    await redis.set(key, JSON.stringify(session))
+
+                    setTimeout(() => setUnverified(r.uuid), config.core.verifyTimeout)
+
+                    console.log(`[Camera] Verified user ${r.uuid}`)
+                }))
+            }
+        }
+    } catch (e) {
+        console.error('camera run error:', e)
+    }
+    setTimeout(run, 1000)
+}
+
+export { run as initCamera };