facenet | Face recognition using Tensorflow | Computer Vision library

def face_match(img_path, data_path): # img_path= location of photo, data_path= location of data.pt 
    # getting embedding matrix of the given img
    img_path = (os.getcwd()+'/1.jpg')
    img = Image.open(img_path)
    face = mtcnn(img) # returns cropped face and probability
    emb = resnet(face.unsqueeze(0)).detach() # detech is to make required gradient false

    saved_data = torch.load('data.pt') # loading data.pt file
    embedding_list = saved_data[0] # getting embedding data
    name_list = saved_data[1] # getting list of names
    dist_list = [] # list of matched distances, minimum distance is used to identify the person
    
    for idx, emb_db in enumerate(embedding_list):
        dist = torch.dist(emb, emb_db)
        dist_list.append(dist)
    
    namestodistance = list(zip(name_list,dist_list))
    
    print(namestodistance)

face_match('1.jpg', 'data.pt')

prototxtPath = r"face_detector\deploy.prototxt"
weightsPath = r"face_detector\res10_300x300_ssd_iter_140000.caffemodel"
faceNet = cv2.dnn.readNet(prototxtPath, weightsPath)

maskNet = load_model("mask_detector.model")

print("[INFO] starting video stream...")
vs = VideoStream(src=0).start()

---------------------------------------------------------------------------
error                                     Traceback (most recent call last)
~\AppData\Local\Temp/ipykernel_13672/2145281415.py in <module>
     34 prototxtPath = r"face_detector\deploy.prototxt"
     35 weightsPath = r"face_detector\res10_300x300_ssd_iter_140000.caffemodel"
---> 36 faceNet = cv2.dnn.readNet(prototxtPath, weightsPath)
     37 
     38 maskNet = load_model("mask_detector.model")

error: OpenCV(4.5.4) D:\a\opencv-python\opencv-python\opencv\modules\dnn\src\caffe\caffe_io.cpp:1126: error: (-2:Unspecified error) FAILED: fs.is_open(). Can't open "face_detector\deploy.prototxt" in function 'cv::dnn::ReadProtoFromTextFile'

File "/usr/local/lib/python3.8/dist-packages/deepface/commons/functions.py", line 51, in initializeFolder, referer: http://127.0.0.1/flaskapp
[wsgi:error] [pid 61915:tid 139981666330368]     os.mkdir(home+"/.deepface"), referer: http://127.0.0.1/flaskapp
[[wsgi:error] [pid 61915:tid 139981666330368] PermissionError: [Errno 13] Permission denied: '/var/www/.deepface', referer: http://127.0.0.1/flaskapp

from flask import Flask,request
from deepface import DeepFace

app = Flask(__name__)


@app.route("/")
def helloworld():
  return "helloworld"


@app.route("/verify",methods=['POST'])
def hello():
    
   # print(request.form.get('base1'))
    base1 = request.form.get('base1')
    base2 = request.form.get('base2')

    return DeepFace.verify(base1, base2,model_name='Facenet')


if __name__ == "__main__":
  app.run()

def verify(img, frame, model):
    results= DeepFace.verify(img, frame, enforce_detection=False, model_name=model)
    print("result: ", results)
    verification= results['verified']
    if verification is True:
        print(model, "Successfully recognised. SCORE=")
        return
    return

    raise ValueError("Invalid arguments passed to verify function: ", instance)
ValueError: ('Invalid arguments passed to verify function: ', <PIL.Image.Image image mode=RGB size=160x160 at 0x7F4D5D03FBE0>)

picture= "extracted_face_picture/single_face_picture.jpg"
picture= Image.open(picture)
picture= picture.resize((160,160))

frames_from_npz= "video_faces.npz"
frames= np.load(frames_from_npz)
frames= frames["arr_0"]
i=0
for frames_arr in frames:
    frame= Image.fromarray(frames_arr)

    df.verify(picture,frame, "Facenet")
    print(i, "Above reuslts are for frame", frame_num)
    
    i+= 1

vs = cv2.VideoCapture(0)
while True:
   Frame = vs.read()

bounding_box = {'top': 0, 'left': 0, 'width': 1920, 'height': 1080}
sct = mss()
while True:
    sct_img = sct.grab(bounding_box)
    frame = np.array(sct_img)

# import the necessary packages
import tensorflow as tf
from tensorflow.keras.applications.mobilenet_v2 import preprocess_input
from tensorflow.keras.preprocessing.image import img_to_array
from tensorflow.keras.models import load_model
from imutils.video import VideoStream
import numpy as np
import imutils
import time
import cv2
import os
from mss import mss
from PIL import Image


def detect_and_predict_mask(frame, faceNet, maskNet):
    # grab the dimensions of the frame and then construct a blob
    # from it
    (h, w) = frame.shape[:2]
    blob = cv2.dnn.blobFromImage(frame, 1.0, (224, 224),
                                 (104.0, 177.0, 123.0))


    # pass the blob through the network and obtain the face detections
    faceNet.setInput(blob)
    detections = faceNet.forward()

    # initialize our list of faces, their corresponding locations,
    # and the list of predictions from our face mask network
    faces = []
    locs = []
    preds = []

    # loop over the detections
    for i in range(0, detections.shape[2]):
        # extract the confidence (i.e., probability) associated with
        # the detection
        confidence = detections[0, 0, i, 2]

        # filter out weak detections by ensuring the confidence is
        # greater than the minimum confidence
        if confidence > 0.5:
            # compute the (x, y)-coordinates of the bounding box for
            # the object
            box = detections[0, 0, i, 3:7] * np.array([w, h, w, h])
            (startX, startY, endX, endY) = box.astype("int")

            # ensure the bounding boxes fall within the dimensions of
            # the frame
            (startX, startY) = (max(0, startX), max(0, startY))
            (endX, endY) = (min(w - 1, endX), min(h - 1, endY))

            # extract the face ROI, convert it from BGR to RGB channel
            # ordering, resize it to 224x224, and preprocess it
            face = frame[startY:endY, startX:endX]
            face = cv2.cvtColor(face, cv2.COLOR_BGR2RGB)
            face = cv2.resize(face, (224, 224))
            face = img_to_array(face)
            face = preprocess_input(face)

            # add the face and bounding boxes to their respective
            # lists
            faces.append(face)
            locs.append((startX, startY, endX, endY))

    # only make a predictions if at least one face was detected
    if len(faces) > 0:
    
        # for faster inference we'll make batch predictions on *all*
        # faces at the same time rather than one-by-one predictions
        # in the above `for` loop
        faces = np.array(faces, dtype="float32")
        preds = maskNet.predict(faces, batch_size=32)
        


    # return a 2-tuple of the face locations and their corresponding
    # locations
    return (locs, preds)


# load our serialized face detector model from disk
prototxtPath = r"face_detector\deploy.prototxt"
weightsPath = r"face_detector\res10_300x300_ssd_iter_140000.caffemodel"
faceNet = cv2.dnn.readNet(prototxtPath, weightsPath)
# load the face mask detector model from disk
maskNet = load_model("mask_detector.model")

# initialize the video stream
print("[INFO] starting video stream...")


bounding_box = {'top': 0, 'left': 0, 'width': 1920, 'height': 1080}

sct = mss()
# loop over the frames from the video stream
while True:
    sct_img = sct.grab(bounding_box)

    # grab the frame from the threaded video stream and resize it
    # to have a maximum width of 400 pixels
    frame = np.array(sct_img)
    #frame = imutils.resize(frame, width=1920, height=1080)

    # detect faces in the frame and determine if they are wearing a
    # face mask or not
    (locs, preds) = detect_and_predict_mask(frame, faceNet, maskNet)

    # loop over the detected face locations and their corresponding
    # locations
    for (box, pred) in zip(locs, preds):
        # unpack the bounding box and predictions
        (startX, startY, endX, endY) = box
        (mask, withoutMask) = pred

        # determine the class label and color we'll use to draw
        # the bounding box and text
        label = "Mask" if mask > withoutMask else "No Mask"
        color = (0, 255, 0) if label == "Mask" else (0, 0, 255)

        # include the probability in the label
        label = "{}: {:.2f}%".format(label, max(mask, withoutMask) * 100)

        # display the label and bounding box rectangle on the output
        # frame
        cv2.putText(frame, label, (startX, startY - 10),
                    cv2.FONT_HERSHEY_SIMPLEX, 0.45, color, 2)
        cv2.rectangle(frame, (startX, startY), (endX, endY), color, 2)

    # show the output frame
    cv2.imshow("Frame", frame)
    key = cv2.waitKey(1) & 0xFF

    # if the `q` key was pressed, break from the loop
    if key == ord("q"):
        break

# do a bit of cleanup
cv2.destroyAllWindows()
vs.stop()

  File "C:\Users\user1\AppData\Roaming\Python\Python36\site-packages\tensorflow_core\contrib\layers\python\layers\initializers.py", line 120, in _initializer
    raise TypeError('Cannot create initializer for non-floating point type.')
TypeError: Cannot create initializer for non-floating point type.

app.post(
    "/uploadFacial",
    multer({
      storage: storage
    }).single('upload'), function(req, res) {
      console.log(req.file.filename);
      var userid = req.body.upload[1];
      console.log("User id: "+userid);

      var success = true;
      var message = '';

      var subStringUserId = userid.substring(1,userid.length-1);
      var dir = "./users/"+subStringUserId;

      //res.setHeader("Content-Type", "application/json");
      
      console.log(dir);
      if (!fs.existsSync(dir)){
        fs.mkdirSync(dir);
        success = false;
        message = 'Cannot detect the person.Please add name in the textbox provided below to save the person.';
        res.status(200).json({message: message,success:success});
        res.end();
      }else {
        console.log("in else");
        var options={
          mode:'text',
          encoding:'utf-8',
          pythonOptions:['-u'],
          scriptPath:'./',
          pythonPath:'',
          args:[subStringUserId,req.file.filename]
        }
        var facialScript = new PythonShell('face_detect.py',options)
        facialScript.on('message',(response)=>{
          console.log(response);
          res.status(200).send(response);
          //res.end();
        
        })

        facialScript.on('error',(err)=>{
          console.log(err);
        })
      }
    });

from deepface import DeepFace
from os.path import join, dirname, realpath
import os, sys
import json


PARENT_DIR = join(dirname(realpath(__file__)), './users/')
TEST_DIR = join(dirname(realpath(__file__)), './uploads/')

print(json.dumps({'success':False,'message':'Cannot detect at moment.Please try later'})) #this is only for debugging
user_id = sys.argv[1] 


img_name = sys.argv[2]


db = os.path.join(PARENT_DIR, user_id)
img = os.path.join(TEST_DIR, img_name)

try:
    df=DeepFace.find(img_path=img,db_path=db, model_name = "Facenet")

    if(df.size==0):
        print(json.dumps({'message': 'Cannot detect the person.Please add name in the textbox provided below to save the person.','success':False}))
    else:
        print(json.dumps({'success':True,'message':os.path.splitext(os.path.basename(df['identity'][0]))[0]}))        
        
         
except Exception as e: 
    #logger.error('Failed to upload to ftp: '+ str(e))
    a = 1

 throw new ERR_HTTP_HEADERS_SENT('set');
    ^

Error [ERR_HTTP_HEADERS_SENT]: Cannot set headers after they are sent to the client
    at ServerResponse.setHeader (_http_outgoing.js:470:11)
    at ServerResponse.header (/Users/priyamvora/Node/Drishti/ImageUploadTest/node_modules/express/lib/response.js:771:10)
    at ServerResponse.send (/Users/priyamvora/Node/Drishti/ImageUploadTest/node_modules/express/lib/response.js:170:12)
    at PythonShell.facialScript.on (/Users/priyamvora/Node/Drishti/ImageUploadTest/multipart.js:129:27)
    at PythonShell.emit (events.js:182:13)
    at /Users/priyamvora/Node/Drishti/ImageUploadTest/node_modules/python-shell/index.js:324:22
    at Array.forEach (<anonymous>)
    at PythonShell.receiveInternal (/Users/priyamvora/Node/Drishti/ImageUploadTest/node_modules/python-shell/index.js:322:15)
    at PythonShell.receive (/Users/priyamvora/Node/Drishti/ImageUploadTest/node_modules/python-shell/index.js:296:21)
    at Socket.emit (events.js:182:13)

source = '0'
webcam = source == '0'
image_size = 640
imgsz = check_img_size(image_size)
# Load the model
filepath = 'weights/mask.pt'
# device = torch.device('cpu')
device = select_device()
# half = device.type != 'cpu'
model = attempt_load(filepath, map_location = device)
model.to(device).eval()
# if half:
#     model.half()

# Second stage classifier 
classify = False
if classify:
    modelc = torch_utils.load_classifier(name = 'resnet101', n = 2)
    modelc.load_state_dict(torch.load('weights/resnet101.pt', map_location = device)['modelc']) ###########
    modelc.to(device).eval()

vid_path, vid_writer = None, None
if webcam:
    view_img = True
    cudnn.benchmark = True
    dataset = LoadStreams(source, img_size = imgsz)
    
    
names = model.module.names if hasattr(model, 'module') else model.names
print(names)


def process_image(image):
    h, w = image.shape[:2]
    desired_size = 416
    ratio = desired_size/w
    print("Ratio",ratio)
    img = cv2.resize(image, (0, 0), fx = ratio, fy = ratio)
    h, w = img.shape[:2]
    img = cv2.copyMakeBorder(img, int((416-h)/2), int((416-h)/2), 0, 0, cv2.BORDER_CONSTANT)
    img = img[:, :, ::-1].transpose(2, 0, 1)
    img = np.ascontiguousarray(img)
    img = torch.from_numpy(img).to(device)
    img = img.float()
    img /=255.0
    if img.ndimension() == 3:
        img = img.unsqueeze(0)
    
    return img

def classify(image):
    # device = torch.device("cpu")
    
    #image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    #im = Image.fromarray(image)
    img = process_image(image)
    print('Image processed')
    # img = image.unsqueeze_(0)
    # img = image.float()
    
    pred = model(img)[0]
    
    # Apply NMS
    pred = non_max_suppression(pred, 0.4, 0.5, classes = [0, 1, 2], agnostic = None )
    if classify:
        pred = apply_classifier(pred, modelc, img, im0s)
    print("1 ", pred)
    
    model.eval()
    model.cpu()

    classification = torch.cat(pred)[:, -1]
    if len(classification) == 0:
        return None
    
    index = int(classification[0])
    print(names[index])
    return names[index]

def detect(frame):
    source = '0'
    webcam = source == '0'
    
    image_size = 640
    imgsz = check_img_size(image_size)
    
    # Load model
    file_path = 'weights/yolov5s.pt'
    #device = torch.device('cpu')
    device = select_device()
    # half = device.type != 'cpu'
    model = attempt_load(file_path, map_location = device)
    model.to(device).eval()
    # if half:
    #     model.half()
    names = model.module.names if hasattr(model, 'module') else model.names
    colors = [[75, 125, 2]]
    img = process_image(frame)
    pred = model(img)[0]
    pred = non_max_suppression(pred, 0.4, 0.5, classes = [0], agnostic = None)
    if classify:
        pred = apply_classifier(pred, modelc, img, im0s)
    gn = torch.tensor(frame.shape)[[1,0,1,0]]    
    for i, det in enumerate(pred):
        det[:,:4] = scale_coords(img.shape[2:], det[:,:4], frame.shape).round()
        
        for *xyxy, conf, cls in reversed(det):
            xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist()  # normalized xywh
            label = '%s %.2f' % (names[int(cls)], conf)
            if label is not None:
                if (label.split())[0] == 'person':
                    plot_one_box(xyxy, frame, label = label, color = colors[0], line_thickness = 1)  # utils.general

with tf.Graph().as_default():
    gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.6)
    sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options, log_device_placement=False))
    with sess.as_default():
        pnet, rnet, onet = detect_face.create_mtcnn(sess, './models/')

        minsize = 20  # minimum size of face
        threshold = [0.6, 0.7, 0.7]  # three steps's threshold
        factor = 0.709  # scale factor
        margin = 44
        frame_interval = 3
        batch_size = 1000
        image_size = 182
        input_image_size = 160

        print('Loading feature extraction model')
        modeldir = './models/'
        facenet.load_model(modeldir)

        images_placeholder = tf.get_default_graph().get_tensor_by_name("input:0")
        embeddings = tf.get_default_graph().get_tensor_by_name("embeddings:0")
        phase_train_placeholder = tf.get_default_graph().get_tensor_by_name("phase_train:0")
        embedding_size = embeddings.get_shape()[1]

        classifier_filename = './myclassifier/my_classifier.pkl'
        classifier_filename_exp = os.path.expanduser(classifier_filename)
        with open(classifier_filename_exp, 'rb') as infile:
            (model, class_names) = pickle.load(infile)
            print('load classifier file-> %s' % type(class_names))
        HumanNames = class_names
        video_capture = cv2.VideoCapture(0)
        c = 0

        print('Start!')
        prevTime = 0
        while True:
            ret, frame = video_capture.read()

            # frame = cv2.resize(frame, (0,0), fx=0.5, fy=0.5)    #resize frame (optional)

            curTime = time.time()    # calc fps
            timeF = frame_interval

            if (c % timeF == 0):
                find_results = []

                if frame.ndim == 2:
                    frame = facenet.to_rgb(frame)
                frame = frame[:, :, 0:3]
                bounding_boxes, _ = detect_face.detect_face(frame, minsize, pnet, rnet, onet, threshold, factor)
                nrof_faces = bounding_boxes.shape[0]
                # print('Bounding Boxes: ', bounding_boxes, 'Shape: ', bounding_boxes.shape, 'nrof_faces:: ', nrof_faces)
                # print('Detected_FaceNum: %d' % nrof_faces)

                if nrof_faces > 0:
                    detect(frame)
                    label = classify(frame)
                    if label == "a":
                        det = bounding_boxes[:, 0:4]
                        img_size = np.asarray(frame.shape)[0:2]
    
                        cropped = []
                        scaled = []
                        scaled_reshape = []
                        bb = np.zeros((nrof_faces,4), dtype=np.int32)
    
                        for i in range(nrof_faces):
                            emb_array = np.zeros((1, embedding_size))
                            # print("Embeddinigs:::::")
                            # print(emb_array)
                            # print("Embeddinigs:::::")
                            bb[i][0] = det[i][0]
                            bb[i][1] = det[i][1]
                            bb[i][2] = det[i][2]
                            bb[i][3] = det[i][3]
    
                            if bb[i][0] <= 0 or bb[i][1] <= 0 or bb[i][2] >= len(frame[0]) or bb[i][3] >= len(frame):
                                print('face is inner of range!')
                                continue
    
                            cropped.append(frame[bb[i][1]:bb[i][3], bb[i][0]:bb[i][2], :])
                            cropped[0] = facenet.flip(cropped[0], False)
                            scaled.append(misc.imresize(cropped[0], (image_size, image_size), interp='bilinear'))
                            scaled[0] = cv2.resize(scaled[0], (input_image_size,input_image_size),
                                                   interpolation=cv2.INTER_CUBIC)
                            scaled[0] = facenet.prewhiten(scaled[0])
                            scaled_reshape.append(scaled[0].reshape(-1,input_image_size,input_image_size,3))
                            feed_dict = {images_placeholder: scaled_reshape[0], phase_train_placeholder: False}
                            emb_array[0, :] = sess.run(embeddings, feed_dict=feed_dict)

                            predictions = model.predict_proba(emb_array)
                            best_class_indices = np.argmax(predictions, axis=1)
                            best_class_probabilities = predictions[np.arange(len(best_class_indices)), best_class_indices]
                            cv2.rectangle(frame, (bb[i][0], bb[i][1]), (bb[i][2], bb[i][3]), (0, 255, 0), 2)
                            text_x = bb[i][0]
                            text_y = bb[i][3] + 20

Traceback (most recent call last):
  File "realtime.py", line 105, in <module>
    label = classify(frame)
  File "yolov5-master\myutils.py", line 117, in classify
    pred = model(img)[0]
  File "C:\Users\Anuj\anaconda3\envs\py36\lib\site-packages\torch\nn\modules\module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "yolov5-master\models\yolo.py", line 122, in forward
    return self.forward_once(x, profile)  # single-scale inference, train
  File "yolov5-master\models\yolo.py", line 138, in forward_once
    x = m(x)  # run
  File "C:\Users\Anuj\anaconda3\envs\py36\lib\site-packages\torch\nn\modules\module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "yolov5-master\models\common.py", line 94, in forward
    return self.conv(torch.cat([x[..., ::2, ::2], x[..., 1::2, ::2], x[..., ::2, 1::2], x[..., 1::2, 1::2]], 1))
  File "C:\Users\Anuj\anaconda3\envs\py36\lib\site-packages\torch\nn\modules\module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "yolov5-master\models\common.py", line 38, in fuseforward
    return self.act(self.conv(x))
  File "C:\Users\Anuj\anaconda3\envs\py36\lib\site-packages\torch\nn\modules\module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "C:\Users\Anuj\anaconda3\envs\py36\lib\site-packages\torch\nn\modules\conv.py", line 419, in forward
    return self._conv_forward(input, self.weight)
  File "C:\Users\Anuj\anaconda3\envs\py36\lib\site-packages\torch\nn\modules\conv.py", line 416, in _conv_forward
    self.padding, self.dilation, self.groups)
RuntimeError: Input type (torch.cuda.FloatTensor) and weight type (torch.FloatTensor) should be the same