Add Live-Processing (todo: get previewlayer working)

intelliScan-analytic-engine/ContentView.swift

@@ -6,50 +6,51 @@
 //
 
 import SwiftUI
+import AVFoundation
 import Vision
 
-struct ContentView: View {
+struct LiveTextRecognitionView: View {
     @State private var recognizedText = ""
-    @State private var showingImagePicker = false
-    @State private var showingCamera = false
-    @State private var selectedImage: UIImage?
 
     var body: some View {
         VStack {
-            Button("Bild auswählen") {
+            CameraView(recognizedText: $recognizedText)
-                self.showingImagePicker.toggle()
+                .edgesIgnoringSafeArea(.all)
-            }
+                .onDisappear {
-            .sheet(isPresented: $showingImagePicker, onDismiss: recognizeText) {
+                    CameraView.stopSession()
-                ImagePicker(selectedImage: self.$selectedImage, sourceType: .photoLibrary)
                 }
 
-            Button("Kamera verwenden") {
+            Text("Live erkannter Text:")
-                self.showingCamera.toggle()
-            }
-            .sheet(isPresented: $showingCamera, onDismiss: recognizeText) {
-                ImagePicker(selectedImage: self.$selectedImage, sourceType: .camera)
-            }
-
-            if let selectedImage = selectedImage {
-                Image(uiImage: selectedImage)
-                    .resizable()
-                    .scaledToFit()
-                    .frame(width: 200, height: 200)
                 .padding()
-
-                Text("Erkannter Text:")
             Text(recognizedText)
                 .padding()
+                .background(Color.white.opacity(0.7))
         }
     }
 }
 
-    func recognizeText() {
+struct LiveTextRecognitionView_Previews: PreviewProvider {
-        guard let selectedImage = selectedImage, let cgImage = selectedImage.cgImage else {
+    static var previews: some View {
-            return
+        LiveTextRecognitionView()
+    }
 }
 
-        let request = VNRecognizeTextRequest(completionHandler: { (request, error) in
+struct CameraView: UIViewRepresentable {
+    @Binding var recognizedText: String
+
+    class Coordinator: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
+        var recognizedText: Binding<String>
+        var request: VNRecognizeTextRequest?
+
+        init(recognizedText: Binding<String>) {
+            self.recognizedText = recognizedText
+            super.init()
+
+            setupVision()
+        }
+
+        func setupVision() {
+            request = VNRecognizeTextRequest(completionHandler: { (request, error) in
                 guard let observations = request.results as? [VNRecognizedTextObservation] else { return }
 
                 var recognizedText = ""
@@ -58,69 +59,84 @@ struct ContentView: View {
                     recognizedText += topCandidate.string + "\n"
                 }
 
-            self.recognizedText = recognizedText
+                self.recognizedText.wrappedValue = recognizedText
             })
 
-        let requestHandler = VNImageRequestHandler(cgImage: cgImage, options: [:])
+            request?.recognitionLevel = .accurate
+        }
+
+        func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
+            guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
+
+            let handler = VNImageRequestHandler(cvPixelBuffer: imageBuffer, options: [:])
 
             do {
-            try requestHandler.perform([request])
+                try handler.perform([request!])
             } catch {
                 print("Error performing OCR: \(error)")
             }
         }
     }
 
-struct ContentView_Previews: PreviewProvider {
+    static var session: AVCaptureSession?
-    static var previews: some View {
+
-        ContentView()
+    static func startSession() {
-    }
+        session?.startRunning()
     }
 
-struct ImagePicker: UIViewControllerRepresentable {
+    static func stopSession() {
-    @Binding var selectedImage: UIImage?
+        session?.stopRunning()
-    var sourceType: UIImagePickerController.SourceType
-
-    class Coordinator: NSObject, UINavigationControllerDelegate, UIImagePickerControllerDelegate {
-        let parent: ImagePicker
-
-        init(parent: ImagePicker) {
-            self.parent = parent
     }
 
-        func imagePickerController(_ picker: UIImagePickerController, didFinishPickingMediaWithInfo info: [UIImagePickerController.InfoKey : Any]) {
-            if let uiImage = info[.originalImage] as? UIImage {
-                parent.selectedImage = uiImage
-            }
-
-            parent.presentationMode.wrappedValue.dismiss()
-        }
-    }
-
-    var imagePickerController: UIImagePickerController
-    @Environment(\.presentationMode) var presentationMode
-
-    init(selectedImage: Binding<UIImage?>, sourceType: UIImagePickerController.SourceType) {
-        _selectedImage = selectedImage
-        self.sourceType = sourceType
-        imagePickerController = UIImagePickerController()
-        imagePickerController.sourceType = sourceType
-        imagePickerController.allowsEditing = false
-    }
-
-    func makeUIViewController(context: Context) -> UIImagePickerController {
-        imagePickerController.delegate = context.coordinator
-        return imagePickerController
-    }
-
-    func updateUIViewController(_ uiViewController: UIImagePickerController, context: Context) {}
-
     func makeCoordinator() -> Coordinator {
-        return Coordinator(parent: self)
+        return Coordinator(recognizedText: $recognizedText)
-    }
     }
 
+    func makeUIView(context: Context) -> UIView {
+        let view = UIView()
 
-#Preview {
+        let session = AVCaptureSession()
-    ContentView()
+
+        guard let device = AVCaptureDevice.default(for: .video) else { return view }
+        let input = try? AVCaptureDeviceInput(device: device)
+
+        if session.canAddInput(input!) {
+            session.addInput(input!)
+        }
+
+        let output = AVCaptureVideoDataOutput()
+        output.setSampleBufferDelegate(context.coordinator, queue: DispatchQueue(label: "cameraQueue"))
+
+        if session.canAddOutput(output) {
+            session.addOutput(output)
+        }
+
+        // Todo: get PreviewLayer working
+        let previewLayer = AVCaptureVideoPreviewLayer(session: session)
+        previewLayer.videoGravity = .resizeAspectFill
+        previewLayer.frame = view.layer.bounds
+        view.layer.addSublayer(previewLayer)
+
+        CameraView.session = session
+
+        return view
+    }
+
+    func updateUIView(_ uiView: UIView, context: Context) {
+        uiView.frame = CGRect(x: 0, y: 0, width: UIScreen.main.bounds.width, height: UIScreen.main.bounds.height)
+
+        if context.coordinator.request == nil {
+            context.coordinator.setupVision()
+        }
+
+        if AVCaptureDevice.authorizationStatus(for: .video) == .authorized {
+            CameraView.startSession()
+        } else {
+            AVCaptureDevice.requestAccess(for: .video) { granted in
+                if granted {
+                    CameraView.startSession()
+                }
+            }
+        }
+    }
 }

intelliScan-analytic-engine/intelliScan_analytic_engineApp.swift

@@ -11,7 +11,7 @@ import SwiftUI
 struct intelliScan_analytic_engineApp: App {
     var body: some Scene {
         WindowGroup {
-            ContentView()
+            LiveTextRecognitionView()
         }
     }
 }