Merge pull request #24 from dkwstas/vision_integration

Vision integration

src/officesense_pi/.gitignore

@@ -2,4 +2,6 @@ node_modules
 # Keep environment variables out of version control
 .env
 
-**/generated/prisma
+/generated/prisma
+
+/generated/prisma

src/officesense_pi/.prettierrc

@@ -0,0 +1,7 @@
+{
+    "semi": true,
+    "singleQuote": false,
+    "tabWidth": 4,
+    "trailingComma": "es5",
+    "printWidth": 100
+}

src/officesense_pi/package-lock.json

@@ -23,6 +23,7 @@
         "@types/express": "^5.0.6",
         "@types/express-session": "^1.19.0",
         "@types/node": "^25.7.0",
+        "prettier": "^3.8.3",
         "tsx": "^4.21.0",
         "typescript": "^6.0.3"
       }
@@ -6653,6 +6654,22 @@
       "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==",
       "license": "MIT"
     },
+    "node_modules/prettier": {
+      "version": "3.8.3",
+      "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.8.3.tgz",
+      "integrity": "sha512-7igPTM53cGHMW8xWuVTydi2KO233VFiTNyF5hLJqpilHfmn8C8gPf+PS7dUT64YcXFbiMGZxS9pCSxL/Dxm/Jw==",
+      "dev": true,
+      "license": "MIT",
+      "bin": {
+        "prettier": "bin/prettier.cjs"
+      },
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "url": "https://github.com/prettier/prettier?sponsor=1"
+      }
+    },
     "node_modules/prisma": {
       "version": "6.19.3",
       "resolved": "https://registry.npmjs.org/prisma/-/prisma-6.19.3.tgz",

src/officesense_pi/package.json

@@ -9,12 +9,15 @@
   "scripts": {
     "dev": "tsx src/index.ts",
     "build": "tsc",
-    "start": "node dist/index.js"
+    "start": "node dist/index.js",
+    "format": "prettier --write \"src/**/*.{ts,tsx}\"",
+    "format:check": "prettier --check \"src/**/*.{ts,tsx}\""
   },
   "devDependencies": {
     "@types/express": "^5.0.6",
     "@types/express-session": "^1.19.0",
     "@types/node": "^25.7.0",
+    "prettier": "^3.8.3",
     "tsx": "^4.21.0",
     "typescript": "^6.0.3"
   },
@@ -29,4 +32,4 @@
     "prisma": "^6.19.3",
     "redis": "^5.12.1"
   }
-}
+}

src/officesense_pi/prisma.config.ts

@@ -1,14 +0,0 @@
-// This file was generated by Prisma, and assumes you have installed the following:
-// npm install --save-dev prisma dotenv
-import "dotenv/config";
-import { defineConfig } from "prisma/config";
-
-export default defineConfig({
-  schema: "prisma/schema.prisma",
-  migrations: {
-    path: "prisma/migrations",
-  },
-  datasource: {
-    url: process.env["DATABASE_URL"],
-  },
-});

src/officesense_pi/src/api/management/components.ts

@@ -8,4 +8,6 @@ export const componentLoader = new ComponentLoader()
 
 export const Components = {
     Dashboard: componentLoader.add('Dashboard', path.join(__dirname, './components/Dashboard.tsx'), 'components'),
+    UploadFace: componentLoader.add('UploadFace', path.join(__dirname, './components/UploadFace.tsx'), 'components'),
+    FaceEmbeddingField: componentLoader.add('FaceEmbeddingField', path.join(__dirname, './components/FaceEmbeddingField.tsx'), 'components'),
 }

src/officesense_pi/src/api/management/components/FaceEmbeddingField.tsx

@@ -0,0 +1,26 @@
+import React from 'react'
+import { BasePropertyProps } from 'adminjs'
+import { Label } from '@adminjs/design-system'
+
+const FaceEmbeddingField: React.FC<BasePropertyProps> = ({ record, property }) => {
+    const has = !!record?.params?.faceEmbedding
+    return (
+        <div style={{ marginBottom: '16px' }}>
+            <Label>{property.label}</Label>
+            <div style={{ marginTop: 6 }}>
+                <span style={{
+                    fontSize: '12px',
+                    fontWeight: 600,
+                    padding: '3px 10px',
+                    borderRadius: '4px',
+                    background: has ? '#dcfce7' : '#fee2e2',
+                    color: has ? '#16a34a' : '#dc2626',
+                }}>
+                    {has ? 'SET' : 'UNSET'}
+                </span>
+            </div>
+        </div>
+    )
+}
+
+export default FaceEmbeddingField

src/officesense_pi/src/api/management/components/UploadFace.tsx

@@ -0,0 +1,96 @@
+import React, { useState, useRef } from 'react'
+import { Box, H3, Text, Button, MessageBox } from '@adminjs/design-system'
+import { ActionProps, useRecord } from 'adminjs'
+
+const UploadFace: React.FC<ActionProps> = ({ record, action }) => {
+    const [preview, setPreview] = useState<string | null>(null)
+    const [base64, setBase64] = useState<string | null>(null)
+    const [loading, setLoading] = useState(false)
+    const [notice, setNotice] = useState<{ message: string, type: 'success' | 'error' } | null>(null)
+    const inputRef = useRef<HTMLInputElement>(null)
+
+    const handleFile = (e: React.ChangeEvent<HTMLInputElement>) => {
+        const file = e.target.files?.[0]
+        if (!file) return
+        const reader = new FileReader()
+        reader.onload = () => {
+            const result = reader.result as string
+            setPreview(result)
+            setBase64(result.split(',')[1]) // strip data:image/jpeg;base64,
+        }
+        reader.readAsDataURL(file)
+    }
+
+    const handleSubmit = async () => {
+        if (!base64) return
+        setLoading(true)
+        setNotice(null)
+        try {
+            const res = await fetch(
+                `/admin/api/resources/User/records/${record?.params.id}/uploadFace`,
+                {
+                    method: 'POST',
+                    headers: { 'Content-Type': 'application/json' },
+                    credentials: 'include',
+                    body: JSON.stringify({ base64 }),
+                }
+            )
+            const data = await res.json()
+            setNotice(data.notice)
+        } catch (e: any) {
+            setNotice({ message: e.message, type: 'error' })
+        } finally {
+            setLoading(false)
+        }
+    }
+
+    return (
+        <Box padding="xl">
+            <H3 marginBottom="md">Upload face image</H3>
+            <Text color="grey60" marginBottom="lg">
+                Upload a JPEG photo of the user to generate and store their face embedding.
+            </Text>
+
+            {notice && (
+                <MessageBox
+                    message={notice.message}
+                    variant={notice.type === 'success' ? 'success' : 'danger'}
+                    marginBottom="lg"
+                />
+            )}
+
+            <Box marginBottom="lg">
+                <input
+                    ref={inputRef}
+                    type="file"
+                    accept="image/jpeg"
+                    onChange={handleFile}
+                    style={{ display: 'none' }}
+                />
+                <Button onClick={() => inputRef.current?.click()} variant="outlined">
+                    Choose JPEG
+                </Button>
+            </Box>
+
+            {preview && (
+                <Box marginBottom="lg">
+                    <img
+                        src={preview}
+                        alt="Preview"
+                        style={{ width: 160, height: 160, objectFit: 'cover', borderRadius: 8, border: '1px solid #e5e7eb' }}
+                    />
+                </Box>
+            )}
+
+            <Button
+                onClick={handleSubmit}
+                disabled={!base64 || loading}
+                variant="contained"
+            >
+                {loading ? 'Processing…' : 'Save embedding'}
+            </Button>
+        </Box>
+    )
+}
+
+export default UploadFace

src/officesense_pi/src/api/management/resources/adminjs.user.resource.ts

@@ -1,5 +1,49 @@
 import { prisma } from "../../../lib/prisma.js";
 import { getModelByName } from "@adminjs/prisma";
+import { spawn } from "child_process";
+import { Components } from '../components.js'
+import { fileURLToPath } from 'url'
+import path from 'path'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+const execPath = '../../../../bin';
+const binaryPath = path.join(__dirname, execPath, 'extractEmbeddings')
+
+async function extractEmbedding(base64: string): Promise<string> {
+    return new Promise((resolve, reject) => {
+        const proc = spawn(binaryPath, [], {
+            cwd: path.join(__dirname, execPath)
+        })
+        let stdout = ''
+        let stderr = ''
+
+        proc.stdout.on('data', (d) => stdout += d)
+        proc.stderr.on('data', (d) => stderr += d)
+
+        proc.stdin.on('error', (err) => { });
+
+        proc.on('error', (err) => {
+            reject(new Error(`Failed to start binary: ${err.message}`))
+        })
+
+        proc.on('close', (code) => {
+            console.log('extractEmbeddings exited with code', code)
+            console.log('stdout:', stdout)
+            console.log('stderr:', stderr)
+            if (code !== 0) return reject(new Error(stderr || `exited with code ${code}`))
+            try {
+                const result = JSON.parse(stdout)
+                if (!result.success) return reject(new Error(result.error))
+                resolve(JSON.stringify(result.descriptor))
+            } catch (e) {
+                reject(new Error(`Failed to parse output: ${stdout}`))
+            }
+        })
+
+        proc.stdin.write(base64)
+        proc.stdin.end()
+    })
+}
 
 export default {
     resource: {
@@ -10,14 +54,56 @@ export default {
         titleProperty: "id",
         navigation: { icon: "User" },
         listProperties: ["id", "firstName", "lastName"],
-        showProperties: ["id", "firstName", "lastName"],
+        showProperties: ["id", "firstName", "lastName", "faceEmbedding"],
         editProperties: ["firstName", "lastName"],
         filterProperties: ["firstName", "lastName"],
         properties: {
             faceEmbedding: {
-                isVisible: false,
+                isVisible: { list: false, show: true, edit: false, filter: false },
-                type: "string"
+                type: "string",
+                components: {
+                    list: Components.FaceEmbeddingField,
+                    show: Components.FaceEmbeddingField,
+                },
             },
-        }
+        },
+        actions: {
+            uploadFace: {
+                actionType: 'record',
+                icon: 'Camera',
+                label: 'Upload face',
+                showInDrawer: true,
+                handler: async (request: any, response: any, context: any) => {
+                    const { record, currentAdmin } = context
+                    if (request.method === 'post') {
+                        const { base64 } = request.payload
+                        if (!base64) {
+                            return {
+                                record: record.toJSON(currentAdmin),
+                                notice: { message: 'No image provided', type: 'error' },
+                            }
+                        }
+                        try {
+                            const embedding = await extractEmbedding(base64)
+                            await prisma.user.update({
+                                where: { id: record.params.id },
+                                data: { faceEmbedding: embedding },
+                            })
+                            return {
+                                record: record.toJSON(currentAdmin),
+                                notice: { message: 'Face embedding saved!', type: 'success' },
+                            }
+                        } catch (e: any) {
+                            return {
+                                record: record.toJSON(currentAdmin),
+                                notice: { message: `Failed: ${e.message}`, type: 'error' },
+                            }
+                        }
+                    }
+                    return { record: record.toJSON(currentAdmin) }
+                },
+                component: Components.UploadFace,
+            },
+        },
     },
 }

src/officesense_pi/src/config/config.ts

@@ -23,6 +23,7 @@ interface Config {
         transitionCleanupInterval: number;
         lossThreshold: number;
         userTTL: number;
+        verifyTimeout: number;
     };
 }
 
@@ -50,7 +51,8 @@ const config: Config = {
         transitionTTL: 5 * 60 * 1000,
         transitionCleanupInterval: 60 * 1000,
         lossThreshold: 5000,
-        userTTL: 3 * 60 * 1000
+        userTTL: 3 * 60 * 1000,
+        verifyTimeout: 30 * 1000
     },
 }
 

src/officesense_pi/src/core/analyze.ts

@@ -38,7 +38,7 @@ export async function updateRoomOccupancyListener(message: string, channel: stri
     const redis = getRedis();
 
     if (!lastRoomID) {
-        console.log("[!] Key not found in local mapι")
+        console.log("[!] Key not found in local map")
     } else {
         await redis.decr(`room:${lastRoomID}`);
     }
@@ -103,8 +103,6 @@ export async function analyzeData(roomID: string, metrics: {
             const user = newUser();
             console.log(`[Redis] Created USER: ${user.psuedoName} USER_ID: ${user.pseudoID}`);
 
-            // trigger camera
-
             await redis.set(`user:${userID}`, JSON.stringify({
                 userID: user.pseudoID,
                 name: user.psuedoName,
@@ -141,8 +139,6 @@ export async function analyzeData(roomID: string, metrics: {
 
         let transition = transitions.get(userID);
 
-        // update room occupancy
-
         if (!transition) {
             console.log("[!] Starting new transition");
             transition = new RoomTransition();
@@ -150,8 +146,6 @@ export async function analyzeData(roomID: string, metrics: {
         }
 
         if (transition.shouldTransitionTo(roomID, metrics.rssi, resObj["rssi"], resObj["timestamp"])) {
-            // trigger camera
-
             console.log(`[!] Transition done ${resObj["room"]} -> ${roomID}`);
 
             await redis.decr(`room:${resObj["room"]}`);
@@ -162,7 +156,7 @@ export async function analyzeData(roomID: string, metrics: {
             resObj["rssi"] = metrics.rssi;
             resObj["room"] = roomID;
             resObj["timestamp"] = Date.now()
-
+            resObj["verified"] = false;
 
             await redis.set(`user:${userID}`, JSON.stringify(resObj), { PX: config.core.userTTL });
         } else

src/officesense_pi/src/core/bootstrap.ts

@@ -1,5 +1,6 @@
 import { getRoomIDs } from "../api/livedata/livedata.repository.js";
 import { initRedis, initSubRedis } from "../redis/redis.js";
+import { initCamera } from "./camera.js";
 import * as mqtt from "./mqtt.js";
 import { cleanupWorker } from "./transition.js";
 
@@ -19,6 +20,8 @@ export async function bootstrap() {
 
         cleanupWorker();
 
+        await initCamera();
+
         console.log("[!] Core started.");
     } catch (err: any) {
         console.log("[!] Bootstrap failed:", err.message);

src/officesense_pi/src/core/camera.ts

@@ -0,0 +1,132 @@
+import { fileURLToPath } from 'url'
+import path from 'path'
+import { spawn } from "child_process";
+import { getRedis } from '../redis/redis.js';
+import { prisma } from '../lib/prisma.js';
+import config from '../config/config.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+const execPath = '../../bin';
+const binaryPath = path.join(__dirname, execPath, 'cameraSession')
+
+type InputPacket = {
+    uuid: string
+    descriptor: number[]
+}
+
+type ResultPacket = {
+    uuid: string
+    verified: boolean
+}
+
+export async function getUnverifiedUserKeys(): Promise<string[]> {
+    const redis = getRedis()
+
+    const keys = await redis.keys('user:*')
+
+    const unverified: string[] = []
+
+    await Promise.all(keys.map(async (key) => {
+        const raw = await redis.get(key)
+        if (!raw) return
+        const session = JSON.parse(raw)
+        if (!session.verified) unverified.push(key.replace('user:', ''))
+    }))
+
+    return unverified
+}
+
+export async function getUnverifiedUsersWithEmbeddings(): Promise<{ uuid: string, descriptor: number[] }[]> {
+    const userIds = await getUnverifiedUserKeys()
+
+    const users = await prisma.user.findMany({
+        where: {
+            id: { in: userIds },
+            faceEmbedding: { not: null },
+        },
+        select: { id: true, faceEmbedding: true },
+    })
+
+    return users.map((user) => ({
+        uuid: user.id,
+        descriptor: JSON.parse(user.faceEmbedding!),
+    }))
+}
+
+async function cameraSession(packets: InputPacket[]): Promise<ResultPacket[] | null> {
+    return new Promise((resolve, reject) => {
+        const proc = spawn(binaryPath, [], {
+            cwd: path.join(__dirname, execPath)
+        })
+        let stdout = ''
+        let stderr = ''
+
+        proc.stdout.on('data', (d) => stdout += d)
+        proc.stderr.on('data', (d) => stderr += d)
+        proc.stdin.on('error', () => { })
+
+        proc.on('error', (err) => {
+            reject(new Error(`Failed to start binary: ${err.message}`))
+        })
+
+        proc.on('close', (code) => {
+            console.log('cameraSession exited with code', code)
+            const trimmed = stdout.trim()
+            if (trimmed === 'no face found.') return resolve(null)
+            if (code !== 0) return reject(new Error(stderr || `exited with code ${code}`))
+            try {
+                const results: ResultPacket[] = JSON.parse(trimmed)
+                resolve(results)
+            } catch (e) {
+                reject(new Error(`Failed to parse output: ${trimmed}`))
+            }
+        })
+
+        for (const packet of packets) {
+            proc.stdin.write(JSON.stringify(packet) + '\n')
+        }
+        proc.stdin.end()
+    })
+}
+
+const setUnverified = async (uuid: string) => {
+    const redis = getRedis()
+    const key = `user:${uuid}`
+    const raw = await redis.get(key)
+    if (!raw) return
+    const session = JSON.parse(raw)
+    session.verified = false
+    await redis.set(key, JSON.stringify(session))
+    console.log(`[Camera] Unverified user ${uuid} due to timeout`)
+}
+
+const run = async () => {
+    const redis = getRedis();
+
+    try {
+        const packets = await getUnverifiedUsersWithEmbeddings()
+        if (packets.length > 0) {
+            const results = await cameraSession(packets)
+            if (results) {
+                await Promise.all(results.map(async (r) => {
+                    if (!r.verified) return
+                    const key = `user:${r.uuid}`
+                    const raw = await redis.get(key)
+                    if (!raw) return
+                    const session = JSON.parse(raw)
+                    session.verified = true
+                    await redis.set(key, JSON.stringify(session))
+
+                    setTimeout(() => setUnverified(r.uuid), config.core.verifyTimeout)
+
+                    console.log(`[Camera] Verified user ${r.uuid}`)
+                }))
+            }
+        }
+    } catch (e) {
+        console.error('camera run error:', e)
+    }
+    setTimeout(run, 1000)
+}
+
+export { run as initCamera };