From 6e5da4553ba94433c99209ef5cf6c68a92cf4153 Mon Sep 17 00:00:00 2001
From: Adwaith Rao <adwaithr99@gmail.com>
Date: Sat, 10 Apr 2021 10:51:44 +0530
Subject: [PATCH] [EXPERIMENTAL] Added code baseline to get it working with
 preview

---
 package-lock.json                       |  2 +
 src/app/ar-fan-cam/ar-fan-cam.page.html |  1 +
 src/app/ar-fan-cam/ar-fan-cam.page.scss |  2 +-
 src/app/ar-fan-cam/ar-fan-cam.page.ts   | 74 ++++++++++++++++++++-----
 4 files changed, 64 insertions(+), 15 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 351a708..f18f124 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -24819,6 +24819,8 @@
       "integrity": "sha512-cOyGQgMdhnRYtW2xrJUNrNYDjEgwQ+BrE2y93Bwz3h4DJ6vJRLfupemU5N3pbYsUlBHJf0u1j1UGk+NLW4d97g==",
       "dev": true,
       "requires": {
+        "@angular/compiler": "9.0.0",
+        "@angular/core": "9.0.0",
         "app-root-path": "^3.0.0",
         "aria-query": "^3.0.0",
         "axobject-query": "2.0.2",
diff --git a/src/app/ar-fan-cam/ar-fan-cam.page.html b/src/app/ar-fan-cam/ar-fan-cam.page.html
index 3c0dcea..ef2d27b 100644
--- a/src/app/ar-fan-cam/ar-fan-cam.page.html
+++ b/src/app/ar-fan-cam/ar-fan-cam.page.html
@@ -5,6 +5,7 @@
     <div class="container" #containerElement>
         <img #glassesElement class="glass-image" [ngStyle]="glassProperties" src="/assets/ar-accessories/glass.png" alt=""/>
         <video id="playback-video" width="{{ width }}" height="{{ height }}" #videoElement autoPlay></video>
+        <img id="source-image" #sourceImageElement width="{{ width }}" height="{{ height }}" />
         <canvas id="source-canvas" #sourceCanvasElement width="{{ width }}" height="{{ height }}"></canvas>
         <canvas id="result-canvas" #canvasElement width="{{ width }}" height="{{ height }}"></canvas>
     </div>
diff --git a/src/app/ar-fan-cam/ar-fan-cam.page.scss b/src/app/ar-fan-cam/ar-fan-cam.page.scss
index 940e0d1..a27cd10 100644
--- a/src/app/ar-fan-cam/ar-fan-cam.page.scss
+++ b/src/app/ar-fan-cam/ar-fan-cam.page.scss
@@ -58,7 +58,7 @@
     display: block;
 }
 
-#result-canvas, #source-canvas, #three-container {
+#result-canvas, #source-canvas, #source-image, #three-container {
     position: absolute;
     top: 0;
     left: 0;
diff --git a/src/app/ar-fan-cam/ar-fan-cam.page.ts b/src/app/ar-fan-cam/ar-fan-cam.page.ts
index f466518..b1a46a6 100644
--- a/src/app/ar-fan-cam/ar-fan-cam.page.ts
+++ b/src/app/ar-fan-cam/ar-fan-cam.page.ts
@@ -1,5 +1,6 @@
 import { Component, ElementRef, OnInit, ViewChild } from '@angular/core';
 import { detectSingleFace, loadFaceExpressionModel, loadFaceLandmarkModel, loadFaceLandmarkTinyModel, loadFaceRecognitionModel, loadSsdMobilenetv1Model, loadTinyFaceDetectorModel, TinyFaceDetectorOptions } from 'face-api.js';
+import { CameraPreview, CameraPreviewPictureOptions, CameraPreviewOptions, CameraPreviewDimensions } from '@ionic-native/camera-preview/ngx';
 import { Location } from '@angular/common';
 import { ModalController } from '@ionic/angular';
 
@@ -12,6 +13,7 @@ export class ArFanCamPage implements OnInit {
     @ViewChild('videoElement') videoElement: ElementRef<HTMLVideoElement>;
     @ViewChild('glassesElement') glassesElement: ElementRef<HTMLImageElement>;
     @ViewChild('canvasElement') canvasElement: ElementRef<HTMLCanvasElement>;
+    @ViewChild('sourceImageElement') sourceImageElement: ElementRef<HTMLImageElement>;
     @ViewChild('sourceCanvasElement') sourceCanvasElement: ElementRef<HTMLCanvasElement>;
     
     mediaStream: MediaStream|null = null;
@@ -34,8 +36,20 @@ export class ArFanCamPage implements OnInit {
 
     constructor(
         private location: Location,
-        private modalController: ModalController
-    ) { }
+        private modalController: ModalController,
+        private cameraPreview: CameraPreview
+    ) {
+        this.sourceImageElement.nativeElement.onload = () => {
+            const context = this.sourceCanvasElement.nativeElement.getContext('2d');
+            context.drawImage(
+                this.sourceImageElement.nativeElement,
+                0,
+                0,
+                this.sourceCanvasElement.nativeElement.width,
+                this.sourceCanvasElement.nativeElement.height
+            );
+        }
+    }
 
     loadNeuralNetModels = async () => {
         await loadTinyFaceDetectorModel('/assets/weights');
@@ -56,19 +70,38 @@ export class ArFanCamPage implements OnInit {
     }
 
     getCameraStream = async () => {
-        const stream = await window.navigator.mediaDevices.getUserMedia({
-            video: {
-                facingMode: 'user',
-                width: this.width,
-                height: this.height
-            },
-        }).then((stream) => {
-            this.mediaStream = stream;
-            this.videoElement.nativeElement.srcObject = stream;
-        }).catch(err => alert(JSON.stringify(err)));        
+        if (this.cameraPreview) {
+            const cameraPreviewOpts: CameraPreviewOptions = {
+                x: 0,
+                y: 0,
+                width: window.screen.width,
+                height: window.screen.height,
+                camera: 'rear',
+                tapPhoto: true,
+                previewDrag: true,
+                toBack: true,
+                alpha: 1
+              }
+            this.cameraPreview.startCamera(cameraPreviewOpts);
+        } else {
+            const stream = await window.navigator.mediaDevices.getUserMedia({
+                video: {
+                    facingMode: 'user',
+                    width: this.width,
+                    height: this.height
+                },
+            }).then((stream) => {
+                this.mediaStream = stream;
+                this.videoElement.nativeElement.srcObject = stream;
+            }).catch(err => alert(JSON.stringify(err)));        
+        }
     };
 
     stopCameraStream = async () => {
+        if (this.cameraPreview) {
+            this.cameraPreview.stopCamera();
+        }
+
         if (this.mediaStream) {
             this.mediaStream.getVideoTracks().forEach(track => {
                 track.stop();
@@ -116,12 +149,25 @@ export class ArFanCamPage implements OnInit {
         context.clearRect(0, 0, window.innerWidth, window.innerHeight);
     }
 
-    detectAndDrawFace = async () => {
-        const tinyFaceDetectorOptions = new TinyFaceDetectorOptions();
+    async getSnapshotFromPreview() {
+        const base64Image: string = await this.cameraPreview.takeSnapshot();
+        this.sourceImageElement.nativeElement.src = base64Image;
+    }
 
+    getSnapshotFromVideo() {
         const sourceCanvas = this.sourceCanvasElement.nativeElement;
         const context = sourceCanvas.getContext('2d');
         context.drawImage(this.videoElement.nativeElement, 0, 0, sourceCanvas.width, sourceCanvas.height);
+    }
+
+    detectAndDrawFace = async () => {
+        const tinyFaceDetectorOptions = new TinyFaceDetectorOptions();
+
+        if (this.cameraPreview) {
+            await this.getSnapshotFromPreview();
+        } else {
+            this.getSnapshotFromVideo();
+        }
 
         let detectionWithLandmarks = await detectSingleFace(this.sourceCanvasElement.nativeElement, tinyFaceDetectorOptions).withFaceLandmarks(true);