diff --git a/.github/workflows/pr_test_cpu.yml b/.github/workflows/pr_test_cpu.yml index 9b37bd77..ef130604 100644 --- a/.github/workflows/pr_test_cpu.yml +++ b/.github/workflows/pr_test_cpu.yml @@ -78,6 +78,16 @@ jobs: cd enhance python3 -m pytest docker container rm -f deepstack + - name: Test Landmark Detection + run: | + docker run -d --name deepstack -e VISION_LANDMARK=True -p 80:5000 deepquestai/deepstack:cpu + cd tests + export TEST_DATA_DIR=$PWD"/test_data" + export TEST_DEEPSTACK_URL="http://localhost:80" + cd landmarkdetection + python3 -m pytest + docker container rm -f deepstack + windows-build: runs-on: windows-2019 steps: diff --git a/Dockerfile.cpu b/Dockerfile.cpu index 553e940c..01d932b6 100644 --- a/Dockerfile.cpu +++ b/Dockerfile.cpu @@ -42,6 +42,7 @@ COPY ./sharedfiles/facerec-high.model /app/sharedfiles/facerec-high.model COPY ./sharedfiles/scene.pt /app/sharedfiles/scene.pt COPY ./sharedfiles/categories_places365.txt /app/sharedfiles/categories_places365.txt COPY ./sharedfiles/bebygan_x4.pth /app/sharedfiles/bebygan_x4.pth +COPY ./sharedfiles/facelandmark.pth /app/sharedfiles/facelandmark.pth RUN mkdir /app/server COPY ./server /app/server diff --git a/intelligencelayer/shared/facelandmark/config.py b/intelligencelayer/shared/facelandmark/config.py new file mode 100644 index 00000000..35cb3acf --- /dev/null +++ b/intelligencelayer/shared/facelandmark/config.py @@ -0,0 +1,18 @@ +from easydict import EasyDict as edict + +class Config: + MODEL=edict() + MODEL.SCALE=4 + MODEL.IN_CHANNEL=3 + MODEL.OUT_CHANNEL=3 + MODEL.N_FEATURE=64 + MODEL.N_BLOCK=6 + MODEL.DEVICE='cuda' + + + + + + + +config=Config() diff --git a/intelligencelayer/shared/facelandmark/model/facelandmark.pth b/intelligencelayer/shared/facelandmark/model/facelandmark.pth new file mode 100644 index 00000000..90906e16 Binary files /dev/null and b/intelligencelayer/shared/facelandmark/model/facelandmark.pth differ diff --git a/intelligencelayer/shared/facelandmark/network.py b/intelligencelayer/shared/facelandmark/network.py new file mode 100644 index 00000000..0b452599 --- /dev/null +++ b/intelligencelayer/shared/facelandmark/network.py @@ -0,0 +1,210 @@ + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torch.autograd import Variable +from torchvision import transforms + + + +''' Creating the Xception Net Model''' + +''' Depthwise Separable Convolution''' + +class DepthewiseSeperableConv2d(nn.Module): + def __init__(self, input_channels, output_channels, kernel_size, **kwargs): + super(DepthewiseSeperableConv2d, self).__init__() + + self.depthwise = nn.Conv2d(input_channels, input_channels, kernel_size, groups = input_channels, bias = False, **kwargs) + self.pointwise = nn.Conv2d(input_channels, output_channels, 1, bias = False) + + def forward(self, x): + x = self.depthwise(x) + x = self.pointwise(x) + + return x + + + +''' Entry block''' + + +class EntryBlock(nn.Module): + def __init__(self): + super(EntryBlock, self).__init__() + + self.conv1 = nn.Sequential( + nn.Conv2d(1, 32, 3, padding = 1, bias = False), + nn.BatchNorm2d(32), + nn.LeakyReLU(0.2) + ) + + self.conv2 = nn.Sequential( + nn.Conv2d(32, 64, 3, padding = 1, bias = False), + nn.BatchNorm2d(64), + nn.LeakyReLU(0.2) + ) + + self.conv3_residual = nn.Sequential( + DepthewiseSeperableConv2d(64, 64, 3, padding = 1), + nn.BatchNorm2d(64), + nn.LeakyReLU(0.2), + DepthewiseSeperableConv2d(64, 128, 3, padding = 1), + nn.BatchNorm2d(128), + nn.MaxPool2d(3, stride = 2, padding = 1), + ) + + self.conv3_direct = nn.Sequential( + nn.Conv2d(64, 128, 1, stride = 2), + nn.BatchNorm2d(128), + ) + + self.conv4_residual = nn.Sequential( + nn.LeakyReLU(0.2), + DepthewiseSeperableConv2d(128, 128, 3, padding = 1), + nn.BatchNorm2d(128), + nn.LeakyReLU(0.2), + DepthewiseSeperableConv2d(128, 256, 3, padding = 1), + nn.BatchNorm2d(256), + nn.MaxPool2d(3, stride = 2, padding = 1) + ) + + self.conv4_direct = nn.Sequential( + nn.Conv2d(128, 256, 1, stride = 2), + nn.BatchNorm2d(256), + ) + + def forward(self, x): + x = self.conv1(x) + x = self.conv2(x) + + residual = self.conv3_residual(x) + direct = self.conv3_direct(x) + x = residual + direct + + residual = self.conv4_residual(x) + direct = self.conv4_direct(x) + x = residual + direct + + return x + +''' Middle basic Block''' +class MiddleBasicBlock(nn.Module): + def __init__(self): + super(MiddleBasicBlock, self).__init__() + + self.conv1 = nn.Sequential( + nn.LeakyReLU(0.2), + DepthewiseSeperableConv2d(256, 256, 3, padding = 1), + nn.BatchNorm2d(256) + ) + self.conv2 = nn.Sequential( + nn.LeakyReLU(0.2), + DepthewiseSeperableConv2d(256, 256, 3, padding = 1), + nn.BatchNorm2d(256) + ) + self.conv3 = nn.Sequential( + nn.LeakyReLU(0.2), + DepthewiseSeperableConv2d(256, 256, 3, padding = 1), + nn.BatchNorm2d(256) + ) + + def forward(self, x): + residual = self.conv1(x) + residual = self.conv2(residual) + residual = self.conv3(residual) + + return x + residual + + +class MiddleBlock(nn.Module): + def __init__(self, num_blocks): + super().__init__() + + self.block = nn.Sequential(*[MiddleBasicBlock() for _ in range(num_blocks)]) + + def forward(self, x): + x = self.block(x) + + return x + +''' Exit Block''' + +class ExitBlock(nn.Module): + def __init__(self): + super(ExitBlock, self).__init__() + + self.residual = nn.Sequential( + nn.LeakyReLU(0.2), + DepthewiseSeperableConv2d(256, 256, 3, padding = 1), + nn.BatchNorm2d(256), + nn.LeakyReLU(0.2), + DepthewiseSeperableConv2d(256, 512, 3, padding = 1), + nn.BatchNorm2d(512), + nn.MaxPool2d(3, stride = 2, padding = 1) + ) + + self.direct = nn.Sequential( + nn.Conv2d(256, 512, 1, stride = 2), + nn.BatchNorm2d(512) + ) + + self.conv = nn.Sequential( + DepthewiseSeperableConv2d(512, 512, 3, padding = 1), + nn.BatchNorm2d(512), + nn.LeakyReLU(0.2), + DepthewiseSeperableConv2d(512, 1024, 3, padding = 1), + nn.BatchNorm2d(1024), + nn.LeakyReLU(0.2) + ) + + self.dropout = nn.Dropout(0.3) + + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + + def forward(self, x): + direct = self.direct(x) + residual = self.residual(x) + x = direct + residual + + x = self.conv(x) + x = self.avgpool(x) + x = self.dropout(x) + + return x + +''' Xception Model Final''' +class XceptionNet(nn.Module): + def __init__(self, num_middle_blocks = 6): + super(XceptionNet, self).__init__() + + self.entry_block = EntryBlock() + self.middel_block = MiddleBlock(num_middle_blocks) + self.exit_block = ExitBlock() + + self.fc = nn.Linear(1024, 136) + + def forward(self, x): + x = self.entry_block(x) + x = self.middel_block(x) + x = self.exit_block(x) + + x = x.view(x.size(0), -1) + + x = self.fc(x) + + return x + + + + + + + + + + + + + diff --git a/intelligencelayer/shared/facelandmark/preprocess.py b/intelligencelayer/shared/facelandmark/preprocess.py new file mode 100644 index 00000000..851d4ad8 --- /dev/null +++ b/intelligencelayer/shared/facelandmark/preprocess.py @@ -0,0 +1,126 @@ +import os +import numpy as np +from PIL import Image +import cv2 +import matplotlib.pyplot as plt +#from skimage import io, transform +#from tqdm.auto import tqdm + +import torch +#import torchvision +#import torch.nn as nn +#import torch.optim as optim +import random +from math import * +import imutils + +import xml.etree.ElementTree as ET + +from torch.utils.data import Dataset, DataLoader, random_split +from torchvision import transforms, utils +from torchsummary import summary +import torchvision.transforms.functional as TF +from __future__ import print_function, division +import pandas as pd +import time +import copy + + + +''' Creating transforms''' + +class Transforms(): + def __init__(self): + pass + + def rotate(self, image, landmarks, angle): + angle = random.uniform(-angle, +angle) + + transformation_matrix = torch.tensor([ + [+cos(radians(angle)), -sin(radians(angle))], + [+sin(radians(angle)), +cos(radians(angle))] + ]) + + image = imutils.rotate(np.array(image), angle) + + landmarks = landmarks - 0.5 + new_landmarks = np.matmul(landmarks, transformation_matrix) + new_landmarks = new_landmarks + 0.5 + return Image.fromarray(image), new_landmarks + + def resize(self, image, landmarks, img_size): + image = TF.resize(image, img_size) + return image, landmarks + + def color_jitter(self, image, landmarks): + color_jitter = transforms.ColorJitter(brightness=0.3, + contrast=0.3, + saturation=0.3, + hue=0.1) + image = color_jitter(image) + return image, landmarks + + def crop_face(self, image, landmarks, crops): + left = int(crops['left']) + top = int(crops['top']) + width = int(crops['width']) + height = int(crops['height']) + + image = TF.crop(image, top, left, height, width) + + img_shape = np.array(image).shape + landmarks = torch.tensor(landmarks) - torch.tensor([[left, top]]) + landmarks = landmarks / torch.tensor([img_shape[1], img_shape[0]]) + return image, landmarks + + def __call__(self, image, landmarks, crops): + image = Image.fromarray(image) + image, landmarks = self.crop_face(image, landmarks, crops) + image, landmarks = self.resize(image, landmarks, (224, 224)) + image, landmarks = self.color_jitter(image, landmarks) + image, landmarks = self.rotate(image, landmarks, angle=10) + + image = TF.to_tensor(image) + image = TF.normalize(image, [0.5], [0.5]) + return image, landmarks + +''' Create face landmark dataset''' +class FaceLandmarkDataset(Dataset): + def __init__(self, transform=None): + tree=ET.parse('path/to/annotations.xml') + root=tree.getroot() + + self.image_filenames=[] + self.landmarks=[] + self.crops=[] + self.transform=transform + self.root_dir='path/to/images/' + + for filename in root[2]: + self.image_filenames.append(os.path.join(self.root_dir, filename.attrib['file'])) + self.crops.append(filename[0].attrib) + + landmark=[] + + for num in range(68): + x_coord=int(filename[0][num].attrib['x']) + y_coord=int(filename[0][num].attrib['y']) + + landmark.append([x_coord, y_coord]) + self.landmarks.append(landmark) + + self.landmarks=np.array(self.landmarks).astype('float32') + + assert len(self.image_filenames)==len(self.landmarks) + def __len__(self): + return len(self.image_filenames) + + def __getitem__(self, idx): + image=cv2.imread(self.image_filenames[idx],0) + landmarks=self.landmarks[idx] + + if self.transform: + image, landmarks=self.transform(image, landmarks, self.crops[idx]) + + landmarks=landmarks - 0.5 + return image, landmarks \ No newline at end of file diff --git a/intelligencelayer/shared/facelandmark/utils.py b/intelligencelayer/shared/facelandmark/utils.py new file mode 100644 index 00000000..aac441d0 --- /dev/null +++ b/intelligencelayer/shared/facelandmark/utils.py @@ -0,0 +1,40 @@ + +import torch +from torch.nn import DataParallel +from torch.nn.parallel import DistributedDataParallel +import torchvision.transforms.functional as TF +import cv2 +def load_model(model,model_path,strict=True,cpu=False): + if isinstance(model,DataParallel) or isinstance(model,DistributedDataParallel): + model=model.module + if cpu: + loaded_model=torch.load(model_path,map_location='cpu') + else: + loaded_model=torch.load(model_path) + model.load_state_dict(loaded_model,strict=strict) + +def transform_img(image): + image=TF.to_pil_image(image) + image = TF.resize(image, (224, 224)) + image = TF.to_tensor(image) + image = (image - image.min())/(image.max() - image.min()) + image = (2 * image) - 1 + return image.unsqueeze(0) + + +def landmarks_draw(image,img_landmarks): + image=image.copy() + for landmarks, (left, top,height,width) in img_landmarks: + landmarks=landmarks.view(-1,2) + landmarks=(landmarks+0.5) + landmarks=landmarks.numpy() + + for i, (x,y) in enumerate(landmarks, 1): + try: + cv2.circle(image, (int((x * width) + left), int((y * height) + top)), 2, [40, 117, 255], -1) + except: + + pass + return image + + diff --git a/intelligencelayer/shared/landmarkdetection.py b/intelligencelayer/shared/landmarkdetection.py new file mode 100644 index 00000000..bfb22fbd --- /dev/null +++ b/intelligencelayer/shared/landmarkdetection.py @@ -0,0 +1,140 @@ +from intelligencelayer.shared.facelandmark.utils import landmarks_draw, transform_img + +from shared import SharedOptions +from PIL import Image +from io import BytesIO +import numpy as np +import sys +import os +import torch +from multiprocessing import process +import base64 +import time +from threading import Thread +from queue import Queue +import cv2 +from imutils import face_utils +import dlib + +import json + +from facelandmark.utils import load_model + +sys.path.insert(0, os.path.join( + os.path.dirname(os.path.realpath(__file__)), ".")) +if SharedOptions.PROFILE == "windows_native": + sys.path.append(os.path.join(SharedOptions.APP_DIR, "windows_packages")) + +def get_network(model_path): + from facelandmark.config import config + from facelandmark.network import XceptionNet + + # an ugly operation + if 'KERNEL_PATH' in config.MODEL: + config.MODEL.KERNEL_PATH = config.MODEL.KERNEL_PATH.replace('../', '') + + return config, XceptionNet(config) + +SHARED_APP_DIR = SharedOptions.SHARED_APP_DIR +CUDA_MODE = SharedOptions.CUDA_MODE +db = SharedOptions.db +TEMP_PATH = SharedOptions.TEMP_PATH + +model_path=os.path.join( + SHARED_APP_DIR, SharedOptions.SETTINGS.FACELANDMARK_MODEL) + +use_cuda= SharedOptions.CUDA_MODE +use_cpu=False if use_cuda else True + +config, model=get_network(model_path) +if use_cuda: + device=torch.device('cuda') +else: + device=torch.device('cpu') + +model=model.to(device) +load_model(model, model_path, strict=True, cpu=use_cpu) + +scale=config.MODEL.SCALE + +detector=dlib.get_frontal_face_detector() #using the dlib face detector for face detection + + +def run_task(q): + while True: + req_data=q.get() + + try: + with torch.no_grad(): + img_id=req_data['imgid'] + req_id=req_data['reqid'] + req_type=req_data['reqtype'] + img_path=os.path.join(TEMP_PATH, img_id) + + + img=Image.open(img_path) + + + + def inference(frame): + gray=cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) + faces=detector(gray, 1) + + outputs=[] + for (i,face) in enumerate(faces): + (x,y,w,h)=face_utils.rect_to_bb(face) #convert dlib rectangle to bounding boxes + + transformed_img=transform_img(gray) + landmarks_predictions=model(transformed_img.to(device)) + outputs.append(landmarks_predictions.to(device),(x,y,w,h)) + return landmarks_draw(frame, outputs) + # + landmark_arr=inference(img) + + for i in range(68): + x=landmark_arr[0][i][0] + y=landmark_arr[0][i][1] + + detection={ + "x": x, + "y": y, + } + + + + output_response={ + "Sucess":True, + + "predictions":detection, + } + except Exception as e: + output_response={ + "Sucess":False, + "error":"An error occured while processing the request", + "code": 500, + } + finally: + db.set(req_id, json.dumps(output_response)) + if os.path.exists(img_path): + os.remove(img_path) +def landmarkdetection2x(delay:float): + q = Queue(maxsize=0) + t = Thread(target=run_task, args=(q,)) + t.daemon = True + t.start() + while True: + try: + req_data=db.get(db.keys()[0]) + if req_data: + q.put(json.loads(req_data)) + time.sleep(delay) + except Exception as e: + print(e) + time.sleep(delay) + +if __name__ == "__main__": + landmarkdetection2x(SharedOptions.SLEEP_TIME) + + + + diff --git a/intelligencelayer/shared/shared.py b/intelligencelayer/shared/shared.py index a4af5266..6638441e 100644 --- a/intelligencelayer/shared/shared.py +++ b/intelligencelayer/shared/shared.py @@ -20,6 +20,7 @@ def __init__( FACE_LOW, FACE_MODEL, SUPERRESOLUTION_MODEL, + FACELANDMARK_MODEL, ): self.DETECTION_HIGH = DETECTION_HIGH self.DETECTION_MEDIUM = DETECTION_MEDIUM @@ -30,6 +31,7 @@ def __init__( self.FACE_LOW = FACE_LOW self.FACE_MODEL = FACE_MODEL self.SUPERRESOLUTION_MODEL = SUPERRESOLUTION_MODEL + self.LANDMARKDETECTION_MODEL = FACELANDMARK_MODEL class SharedOptions: @@ -79,6 +81,7 @@ class SharedOptions: FACE_LOW=256, FACE_MODEL="face.pt", SUPERRESOLUTION_MODEL="bebygan_x4.pth", + FACELANDMARK_MODEL="facelandmark.pth", ), "desktop_gpu": Settings( DETECTION_HIGH=640, @@ -90,6 +93,7 @@ class SharedOptions: FACE_LOW=256, FACE_MODEL="face.pt", SUPERRESOLUTION_MODEL="bebygan_x4.pth", + FACELANDMARK_MODEL="facelandmark.pth", ), "jetson": Settings( DETECTION_HIGH=416, @@ -134,6 +138,7 @@ class SharedOptions: FACE_LOW=256, FACE_MODEL="face.pt", SUPERRESOLUTION_MODEL="bebygan_x4.pth", + FACELANDMARK_MODEL="facelandmark.pth", ), } diff --git a/server/go.mod b/server/go.mod index c40b9700..84bb23c8 100644 --- a/server/go.mod +++ b/server/go.mod @@ -1,19 +1,35 @@ module deepstack.io/server +go 1.18 + require ( - github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/gin-gonic/gin v1.7.0 - github.com/go-ole/go-ole v1.2.4 // indirect github.com/go-redis/redis v6.15.2+incompatible github.com/imroc/req v0.2.4 github.com/mattn/go-sqlite3 v1.11.0 + github.com/satori/go.uuid v1.2.0 + github.com/shirou/gopsutil v2.18.12+incompatible + golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 +) + +require ( + github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-ole/go-ole v1.2.4 // indirect + github.com/go-playground/locales v0.13.0 // indirect + github.com/go-playground/universal-translator v0.17.0 // indirect + github.com/go-playground/validator/v10 v10.4.1 // indirect + github.com/golang/protobuf v1.3.3 // indirect + github.com/json-iterator/go v1.1.9 // indirect + github.com/leodido/go-urn v1.2.0 // indirect + github.com/mattn/go-isatty v0.0.12 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect github.com/onsi/ginkgo v1.12.0 // indirect github.com/onsi/gomega v1.9.0 // indirect - github.com/satori/go.uuid v1.2.0 - github.com/shirou/gopsutil v2.18.12+incompatible github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect - golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 + github.com/ugorji/go/codec v1.1.7 // indirect golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c // indirect + golang.org/x/sys v0.0.0-20200116001909-b77594299b42 // indirect + gopkg.in/yaml.v2 v2.2.8 // indirect ) diff --git a/server/go.sum b/server/go.sum index d24bc43b..d932c811 100644 --- a/server/go.sum +++ b/server/go.sum @@ -3,7 +3,6 @@ github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrU github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= @@ -61,7 +60,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= diff --git a/server/middlewares/middlewares.go b/server/middlewares/middlewares.go index b80f9f67..66df2a39 100644 --- a/server/middlewares/middlewares.go +++ b/server/middlewares/middlewares.go @@ -153,6 +153,32 @@ func CheckSuperresolution() gin.HandlerFunc { } } +func CheckLandmark() gin.HandlerFunc { + + return func(c *gin.Context) { + + activate_landmark := false + landmark := os.Getenv("VISION_LANDMARK") + + if landmark == "True" { + + activate_landmark = true + + } + if activate_landmark == false { + + response := response.ErrorResponse{Success: false, Error: "Landmark endpoint not activated"} + c.JSON(403, response) + c.Abort() + + } else { + c.Next() + } + + } + +} + func CheckApiKey(sub_data *structures.ActivationData, settings *structures.Settings) gin.HandlerFunc { return func(c *gin.Context) { diff --git a/server/requests/requests.go b/server/requests/requests.go index 85bd932f..72203b69 100644 --- a/server/requests/requests.go +++ b/server/requests/requests.go @@ -50,3 +50,8 @@ type SuperresolutionRequest struct { Reqtype string `json:"reqtype"` Reqid string `json:"reqid"` } + +type FacelandmarkRequest struct { + Imgid string `json:"imgid"` + Reqtype string `json:"reqtype"` + Reqid string `json:"reqid"`} \ No newline at end of file diff --git a/server/response/response.go b/server/response/response.go index 0cf95dea..55eceef1 100644 --- a/server/response/response.go +++ b/server/response/response.go @@ -124,3 +124,12 @@ type SuperresolutionResponse struct { Width int `json:"width"` Height int `json:"height"` } + +type LandmarkDetectionPrediction struct{ + X int `json:"x"` + Y int `json:"y"` +} +type FacelandmarkResponse struct{ + Success bool `json:"success"` + Predictions []LandmarkDetectionPrediction `json:"predictions"` +} diff --git a/server/server.go b/server/server.go index 81805746..26253d6e 100644 --- a/server/server.go +++ b/server/server.go @@ -93,7 +93,7 @@ func scene(c *gin.Context) { return } - break + } else if duration > request_timeout { @@ -107,6 +107,53 @@ func scene(c *gin.Context) { } } + + + + +/// Landmark detection func + + + +func landmarkdetection(c *gin.Context, queue_name string) { + img_id := uuid.NewV4().String() + req_id := uuid.NewV4().String() + + landmark_req:=requests.FacelandmarkRequest{Imgid: img_id, Reqtype: "landmark", Reqid: req_id} + landmark_req_string,_:=json.Marshal(landmark_req) + file,_:= c.FormFile("image") + c.SaveUploadedFile(file, filepath.Join(temp_path, img_id)) + redis_client.RPush(queue_name, landmark_req_string) + t1:= time.Now() + for true{ + output,_:=redis_client.Get(req_id).Result() + duration:=time.Since(t1).Seconds() + + if output != ""{ + var res response.FacelandmarkResponse + json.Unmarshal([]byte(output),&res) + if res.Success == false{ + var error_response response.ErrorResponseInternal + json.Unmarshal([]byte(output),&error_response) + final_response := response.ErrorResponse{Success: false, Error: error_response.Error} + c.JSON(error_response.Code,final_response) + return + + } else { + c.JSON(200, res) + return + } + }else if duration > request_timeout{ + final_reponse:=response.ErrorResponse{ Success: false, Error: "Failed to process request before timeout"} + c.JSON(500, final_reponse) + return + } + time.Sleep(1* time.Millisecond) + } + + +} + func detection(c *gin.Context, queue_name string) { nms := c.PostForm("min_confidence") @@ -160,7 +207,7 @@ func detection(c *gin.Context, queue_name string) { return } - break + } else if duration > request_timeout { final_res := response.ErrorResponse{Success: false, Error: "failed to process request before timeout"} @@ -285,7 +332,7 @@ func facerecognition(c *gin.Context) { return } - break + } else if duration > request_timeout { final_res := response.ErrorResponse{Success: false, Error: "failed to process request before timeout"} @@ -307,7 +354,7 @@ func faceregister(c *gin.Context) { user_images := []string{} if form != nil { - for filename, _ := range form.File { + for filename := range form.File { file, _ := c.FormFile(filename) img_id := uuid.NewV4().String() c.SaveUploadedFile(file, filepath.Join(temp_path, img_id)) @@ -350,7 +397,7 @@ func faceregister(c *gin.Context) { return } - break + } else if duration > request_timeout { final_res := response.ErrorResponse{Success: false, Error: "failed to process request before timeout"} @@ -370,7 +417,7 @@ func facematch(c *gin.Context) { user_images := []string{} if form != nil { - for filename, _ := range form.File { + for filename := range form.File { file, _ := c.FormFile(filename) img_id := uuid.NewV4().String() c.SaveUploadedFile(file, filepath.Join(temp_path, img_id)) @@ -413,7 +460,7 @@ func facematch(c *gin.Context) { return } - break + } else if duration > request_timeout { final_res := response.ErrorResponse{Success: false, Error: "failed to process request before timeout"} @@ -599,7 +646,7 @@ func single_request_loop(c *gin.Context, queue_name string) { } - break + } time.Sleep(1 * time.Millisecond) @@ -750,7 +797,7 @@ func superresolution(c *gin.Context, queue_name string) { return } - break + } else if duration > request_timeout { final_res := response.ErrorResponse{Success: false, Error: "failed to process request before timeout"} @@ -861,6 +908,7 @@ func initActivation() { enhance := os.Getenv("VISION_ENHANCE") api_key := os.Getenv("API_KEY") send_logs := os.Getenv("SEND_LOGS") + landmark:= os.Getenv("VISION_LANDMARK") if os.Getenv("VISION-FACE") == "" { os.Setenv("VISION-FACE", face) @@ -874,6 +922,10 @@ func initActivation() { if os.Getenv("VISION-ENHANCE") == "" { os.Setenv("VISION-ENHANCE", enhance) } + if os.Getenv("VISION_LANDMARK") == "" { + os.Setenv("VISION_LANDMARK", landmark) + + } if os.Getenv("API-KEY") == "" { os.Setenv("API-KEY", api_key) } @@ -902,6 +954,7 @@ func main() { var certPath string var sendLogs string var threadcount int + var LandmarkDetection string if os.Getenv("PROFILE") == "" { os.Chdir("C://DeepStack//server") @@ -935,6 +988,7 @@ func main() { flag.StringVar(&adminKey, "ADMIN-KEY", os.Getenv("ADMIN-KEY"), "admin key to secure admin endpoints") flag.StringVar(&modelStoreDetection, "MODELSTORE-DETECTION", "/modelstore/detection/", "path to custom detection models") flag.StringVar(&certPath, "CERT-PATH", "/cert", "path to ssl certificate files") + flag.StringVar(&LandmarkDetection, "VISION_LANDMARK", os.Getenv("VISION_LANDMARK"), "enable landmark detection") floatTimeoutVal, err := strconv.ParseFloat(os.Getenv("TIMEOUT"), 32) if err != nil { @@ -1007,6 +1061,7 @@ func main() { os.Setenv("APPDIR", APPDIR) os.Setenv("MODE", mode) os.Setenv("THREADCOUNT", strconv.Itoa(threadcount)) + os.Setenv("VISION_LANDMARK", LandmarkDetection) } if DATA_DIR == "" { @@ -1064,6 +1119,7 @@ func main() { faceScript := filepath.Join(APPDIR, "intelligencelayer/shared/face.py") sceneScript := filepath.Join(APPDIR, "intelligencelayer/shared/scene.py") enhanceScript := filepath.Join(APPDIR, "intelligencelayer/shared/superresolution.py") + landmarkScript := filepath.Join(APPDIR, "intelligencelayer/shared/landmarkdetection.py") initcmd := exec.CommandContext(ctx, "bash", "-c", interpreter+" "+initScript) if PROFILE == "windows_native" { @@ -1118,6 +1174,26 @@ func main() { } + + if LandmarkDetection == "True" { + landmarkcmd:= exec.CommandContext(ctx, "bash", "-c", interpreter+" "+landmarkScript) + if PROFILE == "windows_native" { + landmarkcmd = exec.CommandContext(ctx, interpreter, landmarkScript) + } + startedProcesses =append(startedProcesses, landmarkcmd) + landmarkcmd.Dir= filepath.Join(APPDIR, "intelligencelayer/shared") + landmarkcmd.Stdout = stdout + landmarkcmd.Stderr = stderr + landmarkcmd.Env = os.Environ() + + err = landmarkcmd.Start() + if err != nil { + stderr.WriteString("Landmark detection process failed to start" + err.Error()) + } + + + } + if visionFace == "True" { facecmd := exec.CommandContext(ctx, "bash", "-c", interpreter+" "+faceScript) if PROFILE == "windows_native" { @@ -1226,6 +1302,12 @@ func main() { superresolution(c, "superresolution_queue") + }) + /// adding landmark detection endpoint + vision.POST("/landmark", middlewares.CheckLandmark(), middlewares.CheckImage(), func (c*gin.Context){ + + landmarkdetection(c, "landmark_queue") + }) facegroup := vision.Group("/face")