Add Live-Processing (todo: get previewlayer working)

intelliScan-analytic-engine/ContentView.swift

@@ -6,121 +6,137 @@
 //
 
 import SwiftUI
+import AVFoundation
 import Vision
 
-struct ContentView: View {
+struct LiveTextRecognitionView: View {
     @State private var recognizedText = ""
-    @State private var showingImagePicker = false
-    @State private var showingCamera = false
-    @State private var selectedImage: UIImage?
 
     var body: some View {
         VStack {
-            Button("Bild auswählen") {
+            CameraView(recognizedText: $recognizedText)
-                self.showingImagePicker.toggle()
+                .edgesIgnoringSafeArea(.all)
-            }
+                .onDisappear {
-            .sheet(isPresented: $showingImagePicker, onDismiss: recognizeText) {
+                    CameraView.stopSession()
-                ImagePicker(selectedImage: self.$selectedImage, sourceType: .photoLibrary)
+                }
-            }
 
-            Button("Kamera verwenden") {
+            Text("Live erkannter Text:")
-                self.showingCamera.toggle()
+                .padding()
-            }
+            Text(recognizedText)
-            .sheet(isPresented: $showingCamera, onDismiss: recognizeText) {
+                .padding()
-                ImagePicker(selectedImage: self.$selectedImage, sourceType: .camera)
+                .background(Color.white.opacity(0.7))
-            }
-
-            if let selectedImage = selectedImage {
-                Image(uiImage: selectedImage)
-                    .resizable()
-                    .scaledToFit()
-                    .frame(width: 200, height: 200)
-                    .padding()
-
-                Text("Erkannter Text:")
-                Text(recognizedText)
-                    .padding()
-            }
-        }
-    }
-
-    func recognizeText() {
-        guard let selectedImage = selectedImage, let cgImage = selectedImage.cgImage else {
-            return
-        }
-
-        let request = VNRecognizeTextRequest(completionHandler: { (request, error) in
-            guard let observations = request.results as? [VNRecognizedTextObservation] else { return }
-
-            var recognizedText = ""
-            for observation in observations {
-                guard let topCandidate = observation.topCandidates(1).first else { continue }
-                recognizedText += topCandidate.string + "\n"
-            }
-
-            self.recognizedText = recognizedText
-        })
-
-        let requestHandler = VNImageRequestHandler(cgImage: cgImage, options: [:])
-
-        do {
-            try requestHandler.perform([request])
-        } catch {
-            print("Error performing OCR: \(error)")
         }
     }
 }
 
-struct ContentView_Previews: PreviewProvider {
+struct LiveTextRecognitionView_Previews: PreviewProvider {
     static var previews: some View {
-        ContentView()
+        LiveTextRecognitionView()
     }
 }
 
-struct ImagePicker: UIViewControllerRepresentable {
+struct CameraView: UIViewRepresentable {
-    @Binding var selectedImage: UIImage?
+    @Binding var recognizedText: String
-    var sourceType: UIImagePickerController.SourceType
 
-    class Coordinator: NSObject, UINavigationControllerDelegate, UIImagePickerControllerDelegate {
+    class Coordinator: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
-        let parent: ImagePicker
+        var recognizedText: Binding<String>
+        var request: VNRecognizeTextRequest?
 
-        init(parent: ImagePicker) {
+        init(recognizedText: Binding<String>) {
-            self.parent = parent
+            self.recognizedText = recognizedText
+            super.init()
+
+            setupVision()
         }
 
-        func imagePickerController(_ picker: UIImagePickerController, didFinishPickingMediaWithInfo info: [UIImagePickerController.InfoKey : Any]) {
+        func setupVision() {
-            if let uiImage = info[.originalImage] as? UIImage {
+            request = VNRecognizeTextRequest(completionHandler: { (request, error) in
-                parent.selectedImage = uiImage
+                guard let observations = request.results as? [VNRecognizedTextObservation] else { return }
+
+                var recognizedText = ""
+                for observation in observations {
+                    guard let topCandidate = observation.topCandidates(1).first else { continue }
+                    recognizedText += topCandidate.string + "\n"
+                }
+
+                self.recognizedText.wrappedValue = recognizedText
+            })
+
+            request?.recognitionLevel = .accurate
+        }
+
+        func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
+            guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
+
+            let handler = VNImageRequestHandler(cvPixelBuffer: imageBuffer, options: [:])
+
+            do {
+                try handler.perform([request!])
+            } catch {
+                print("Error performing OCR: \(error)")
             }
-
-            parent.presentationMode.wrappedValue.dismiss()
         }
     }
 
-    var imagePickerController: UIImagePickerController
+    static var session: AVCaptureSession?
-    @Environment(\.presentationMode) var presentationMode
 
-    init(selectedImage: Binding<UIImage?>, sourceType: UIImagePickerController.SourceType) {
+    static func startSession() {
-        _selectedImage = selectedImage
+        session?.startRunning()
-        self.sourceType = sourceType
-        imagePickerController = UIImagePickerController()
-        imagePickerController.sourceType = sourceType
-        imagePickerController.allowsEditing = false
     }
 
-    func makeUIViewController(context: Context) -> UIImagePickerController {
+    static func stopSession() {
-        imagePickerController.delegate = context.coordinator
+        session?.stopRunning()
-        return imagePickerController
     }
 
-    func updateUIViewController(_ uiViewController: UIImagePickerController, context: Context) {}
-
     func makeCoordinator() -> Coordinator {
-        return Coordinator(parent: self)
+        return Coordinator(recognizedText: $recognizedText)
+    }
+
+    func makeUIView(context: Context) -> UIView {
+        let view = UIView()
+
+        let session = AVCaptureSession()
+
+        guard let device = AVCaptureDevice.default(for: .video) else { return view }
+        let input = try? AVCaptureDeviceInput(device: device)
+
+        if session.canAddInput(input!) {
+            session.addInput(input!)
+        }
+
+        let output = AVCaptureVideoDataOutput()
+        output.setSampleBufferDelegate(context.coordinator, queue: DispatchQueue(label: "cameraQueue"))
+
+        if session.canAddOutput(output) {
+            session.addOutput(output)
+        }
+
+        // Todo: get PreviewLayer working
+        let previewLayer = AVCaptureVideoPreviewLayer(session: session)
+        previewLayer.videoGravity = .resizeAspectFill
+        previewLayer.frame = view.layer.bounds
+        view.layer.addSublayer(previewLayer)
+
+        CameraView.session = session
+
+        return view
+    }
+
+    func updateUIView(_ uiView: UIView, context: Context) {
+        uiView.frame = CGRect(x: 0, y: 0, width: UIScreen.main.bounds.width, height: UIScreen.main.bounds.height)
+
+        if context.coordinator.request == nil {
+            context.coordinator.setupVision()
+        }
+
+        if AVCaptureDevice.authorizationStatus(for: .video) == .authorized {
+            CameraView.startSession()
+        } else {
+            AVCaptureDevice.requestAccess(for: .video) { granted in
+                if granted {
+                    CameraView.startSession()
+                }
+            }
+        }
     }
 }
-
-
-#Preview {
-    ContentView()
-}

intelliScan-analytic-engine/intelliScan_analytic_engineApp.swift

@@ -11,7 +11,7 @@ import SwiftUI
 struct intelliScan_analytic_engineApp: App {
     var body: some Scene {
         WindowGroup {
-            ContentView()
+            LiveTextRecognitionView()
         }
     }
 }