detect_structures.py 33 KB
Newer Older
tilman's avatar
tilman committed
1
2
3
4
5
6
7

# From Python
import sys
import cv2
import numpy as np
import os
from sys import platform
8
from dotmap import DotMap
tilman's avatar
tilman committed
9
10
from shapely import affinity
from shapely.geometry import Polygon
tilman's avatar
tilman committed
11
12
13
14
15
#sys.path.append('/Users/Tilman/Documents/Programme/Python/forschungspraktikum/openpose/python');
from openpose import pyopenpose as op
from lib.triangles import *
from lib.bisection import *
from lib.misc import *
tilman's avatar
tilman committed
16
from lib.kmeans import *
tilman's avatar
tilman committed
17

tilman's avatar
tilman committed
18
SHOW_WINDOW = False
19
SAVE_FILE = not SHOW_WINDOW
20
SAVE_BDCN = False
tilman's avatar
tilman committed
21
BDCN_EDGES = False
22
23
TRIANGLES = False
BODY_LINES = True
tilman's avatar
tilman committed
24
BISEC_VECTORS = False
tilman's avatar
tilman committed
25
BISEC_CONES = True
tilman's avatar
tilman committed
26
GLOBAL_LINE = True
tilman's avatar
tilman committed
27
INPAINT_AND_KMEANS = True
28
DRAW_FIRST_CONE_LAYER_BRIGTHER = True
29
30
BISEC_CONE_ANGLE = 50
CORRECTION_ANGLE = 23
tilman's avatar
tilman committed
31
OVERLAY_ALPHA = 0.2
tilman's avatar
tilman committed
32
COLORED_CANVAS = True
33
BISEC_SKIP_LOWER_LEVELS = False
34
DISPLAY_RASTER_ELEMENTS = 500
35
KMEANS_AREA_MASK_THRESHOLD = 0.08 #0.08 #max 1.0 (percent of pixels), cur 0.09,  sonntag mittag 0.05,   #smaller threshold -> more colors, higher threshold -> less colors
36
KMEANS_K = 7 #10
tilman's avatar
tilman committed
37
38
#KMEANS_K = 6 #10
OUT_DIR = 'images/out/images_imdahl/kmeans_output_canvas/final'
tilman's avatar
tilman committed
39
#IN_DIR = "images/first_email/"     # images from first email
tilman's avatar
tilman committed
40
IN_DIR = "images/images_imdahl/"    # images from imdahl
tilman's avatar
tilman committed
41
IN_DIR_BDCN = "images/out/images_imdahl/bdcn_output/"
tilman's avatar
tilman committed
42

tilman's avatar
tilman committed
43
SKIP_OPENPOSE = False
44
45
46
47
48
49
50
51
OPENPOSE_DEMO_KEYPOINTS = np.array([[[4.7613028e+02, 3.3695804e+02, 9.0203685e-01],[5.3667474e+02, 3.8633786e+02, 6.6615295e-01],[5.1645105e+02, 3.8405157e+02, 5.1514143e-01],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[5.5459924e+02, 3.8859457e+02, 6.4240879e-01],[5.6353766e+02, 4.7384988e+02, 1.8810490e-01],[5.3886292e+02, 5.2543573e+02, 9.0144195e-02],[5.4566248e+02, 5.3215259e+02, 3.6083767e-01],[5.2768524e+02, 5.3213129e+02, 3.1196830e-01],[5.4556714e+02, 6.3534674e+02, 1.8182488e-01],[5.8149310e+02, 7.2958716e+02, 1.3625422e-01],[5.6579541e+02, 5.3216382e+02, 3.6866242e-01],[5.8822272e+02, 6.2862476e+02, 1.7708556e-01],[6.0843213e+02, 7.2955762e+02, 2.2736737e-01],[4.7597812e+02, 3.2798129e+02, 5.7176876e-01],[4.8729745e+02, 3.3027243e+02, 9.1296065e-01],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[5.2090784e+02, 3.3472034e+02, 7.7942842e-01],[5.7928674e+02, 7.5646222e+02, 2.0351715e-01],[5.9049512e+02, 7.5648248e+02, 2.0819387e-01],[6.2183606e+02, 7.3853394e+02, 1.7312977e-01],[5.8145673e+02, 7.5420642e+02, 1.2660497e-01],[5.7701074e+02, 7.5417773e+02, 1.2881383e-01],[5.8374255e+02, 7.3627380e+02, 9.4869599e-02]]
                                    ,[[6.4435681e+02, 3.6383255e+02, 8.9096022e-01],[6.6903070e+02, 3.9760306e+02, 8.7681645e-01],[6.4430103e+02, 3.9525812e+02, 7.9584122e-01],[6.3310535e+02, 4.5589160e+02, 3.7108111e-01],[5.9046979e+02, 4.2451276e+02, 4.0277350e-01],[6.9366602e+02, 4.0197583e+02, 8.9528430e-01],[6.8247137e+02, 4.6042902e+02, 5.5132395e-01],[6.0616620e+02, 4.3569894e+02, 3.4303352e-01],[6.5551196e+02, 5.1196445e+02, 2.9572365e-01],[6.3529651e+02, 5.0747903e+02, 2.8629595e-01],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[6.7573169e+02, 5.1421967e+02, 3.0180413e-01],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[6.4206000e+02, 3.5276721e+02, 7.2430253e-01],[6.5327673e+02, 3.5271103e+02, 9.4265050e-01],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[6.7577380e+02, 3.5269864e+02, 8.9672232e-01],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00]]
                                    ,[[7.2723553e+02, 4.0875150e+02, 8.3982950e-01],[7.6091986e+02, 4.6032086e+02, 5.0676465e-01],[7.3178253e+02, 4.5359366e+02, 3.5797939e-01],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[7.8784674e+02, 4.6483188e+02, 5.6356871e-01],[7.6320721e+02, 5.6802844e+02, 3.7939239e-01],[7.2953772e+02, 5.4564911e+02, 1.5424372e-01],[7.6546356e+02, 6.1964557e+02, 1.7308682e-01],[7.3854327e+02, 6.1513757e+02, 1.5351829e-01],[7.3855487e+02, 7.3405249e+02, 5.6986582e-02],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[7.8789227e+02, 6.2636108e+02, 1.8666090e-01],[7.9010718e+02, 7.5197815e+02, 9.0752751e-02],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[7.2722571e+02, 3.9980579e+02, 4.9854943e-01],[7.4074554e+02, 4.0420221e+02, 8.2562774e-01],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[7.6537799e+02, 4.0880304e+02, 6.8228495e-01],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00]]
                                    ,[[2.6297342e+02, 3.4823679e+02, 9.1535652e-01],[2.1584425e+02, 3.8410617e+02, 4.2777365e-01],[2.0466562e+02, 3.8629623e+02, 6.5148002e-01],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[2.2483388e+02, 3.7963403e+02, 2.8349286e-01],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[2.1584836e+02, 5.5681036e+02, 7.1318626e-02],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[2.5625162e+02, 3.3922253e+02, 8.9375269e-01],[2.6528430e+02, 3.3701016e+02, 1.3707811e-01],[2.2490630e+02, 3.4151849e+02, 8.1041366e-01],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00]]
                                    ,[[3.1685654e+02, 3.3244104e+02, 6.7855740e-01],[2.9669766e+02, 3.7735825e+02, 3.5962355e-01],[2.6300262e+02, 3.8186972e+02, 4.8755571e-01],[2.8984323e+02, 5.1421375e+02, 1.4892229e-01],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[3.3024896e+02, 3.7068640e+02, 3.1298172e-01],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[3.0774149e+02, 3.2796570e+02, 6.2570477e-01],[3.1678952e+02, 3.1911349e+02, 2.6238269e-01],[2.8093988e+02, 3.3702823e+02, 4.3097427e-01],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00]]
                                    ,[[4.0661322e+02, 3.3243243e+02, 7.2449613e-01],[3.5496320e+02, 3.7965060e+02, 3.2941282e-01],[2.0466562e+02, 3.8629623e+02, 6.5148002e-01],[2.4725473e+02, 4.3794165e+02, 1.0388593e-01],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[3.9527917e+02, 3.7732455e+02, 1.9104436e-01],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[3.9753812e+02, 3.2572601e+02, 7.9601538e-01],[4.1098145e+02, 3.2347913e+02, 4.7544584e-01],[3.5937631e+02, 3.2570648e+02, 6.8124008e-01],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00]]
                                    ,[[1.1046178e+02, 3.3481174e+02, 8.0748719e-01],[5.2089359e+01, 3.7064417e+02, 1.8357244e-01],[6.7774979e+01, 3.6842288e+02, 4.0538907e-01],[1.3283961e+02, 3.8408841e+02, 2.2997330e-01],[1.7771373e+02, 3.4149902e+02, 2.7701011e-01],[3.4127533e+01, 3.6390732e+02, 1.1019738e-01],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[9.9306007e+01, 3.3019724e+02, 9.2014235e-01],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[7.8990585e+01, 3.3474988e+02, 8.4556317e-01],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00]]
                                    ,[[7.0929346e+02, 3.5261667e+02, 3.9232758e-01],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[7.8120416e+02, 4.0420923e+02, 5.1513046e-02],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[7.0712549e+02, 3.4816431e+02, 1.5942883e-01],[7.1832990e+02, 3.4808749e+02, 3.8954309e-01],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[7.4299701e+02, 3.5270523e+02, 2.7498546e-01],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00],[0.0000000e+00, 0.0000000e+00, 0.0000000e+00]]])
tilman's avatar
tilman committed
52
53

## misc functions
tilman's avatar
tilman committed
54
def overlay_two_image_v2(image, overlay, ignore_color=[0,0,0], alpha=0.1):
tilman's avatar
tilman committed
55
56
    ignore_color = np.asarray(ignore_color)
    mask = (overlay==ignore_color).all(-1,keepdims=True)
tilman's avatar
tilman committed
57
    out = np.where(mask,image,(image * (1-alpha) + overlay * alpha).astype(image.dtype))
tilman's avatar
tilman committed
58
59
60
    return out
## misc functions end

tilman's avatar
tilman committed
61

tilman's avatar
tilman committed
62
63
64
params = dict()
#params["model_folder"] = "/Users/Tilman/Documents/Programme/Python/forschungspraktikum/openpose/models/"
params["model_folder"] = os.environ['OPENPOSE_MODELS']
65
#https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/doc/output.md#pose-output-format-body_25
tilman's avatar
tilman committed
66
67

# Starting OpenPose
68
69
70
71
if not SKIP_OPENPOSE:
    opWrapper = op.WrapperPython()
    opWrapper.configure(params)
    opWrapper.start()
tilman's avatar
tilman committed
72

tilman's avatar
tilman committed
73
images = [os.path.join(os.getcwd(), IN_DIR, f) for f in os.listdir(IN_DIR)] #make path absolute so os.chdir has no side effects
tilman's avatar
tilman committed
74
75
76
images_bdcn = [os.path.join(os.getcwd(), IN_DIR_BDCN, f) for f in os.listdir(IN_DIR_BDCN)]
images.sort()
images_bdcn.sort()
tilman's avatar
tilman committed
77
78

os.chdir(OUT_DIR) #save images in this dir
tilman's avatar
tilman committed
79
#for img_name in images:
80
# for img_name, img_bdcn in list(zip(images, images_bdcn)):
81
82
for img_name, img_bdcn in list(zip(images, images_bdcn))[0:1]:
# for img_name, img_bdcn in (list(zip(images, images_bdcn))[15:16] if SKIP_OPENPOSE else list(zip(images, images_bdcn))): #jesus
83
# for img_name, img_bdcn in list(zip(images, images_bdcn))[6:7]: #fusswaschung
tilman's avatar
tilman committed
84
85
    # Process Image
    print("calculating: "+img_name)
tilman's avatar
tilman committed
86
    img = cv2.imread(img_name)
tilman's avatar
tilman committed
87
88
    output_canvas = np.array([[[255,255,255]]*len(img[0])]*len(img),np.uint8)
    bdcn_image = cv2.imread(img_bdcn)
89
90
91
92
93
94
95
96
97
98
    max_lw = max(len(img),len(img[0]))
    esz = max_lw / DISPLAY_RASTER_ELEMENTS
    if SKIP_OPENPOSE:
        datum = DotMap()
        datum.poseKeypoints = OPENPOSE_DEMO_KEYPOINTS
        print("Skipping OPENPOSE")
    else:
        datum = op.Datum()
        datum.cvInputData = img
        opWrapper.emplaceAndPop([datum])
tilman's avatar
tilman committed
99
100
        img = datum.cvOutputData
    
tilman's avatar
tilman committed
101
    if INPAINT_AND_KMEANS:
tilman's avatar
tilman committed
102
103
104
105
106
107
        # general idea, people are placed in foreground 
        # -> inpaint around people to replace color invormation from people with color information from direct environment
        # -> kmeans the result to dramatically reduce the amount of colors 
        # -> check witch colors are now on the position of the people. The colors participating more than 8% are considered as foreground and replaced by color with the most participation in foreground
        # -> output this mask if color information with details is needed.
        # -> if binary mask is needed we further do some morph filtering to get away small details and blobs
tilman's avatar
tilman committed
108
        target = cv2.imread(img_name)
tilman's avatar
tilman committed
109
110
111
112
113
114
115
        #remove cracks and prepare for kmeans
        # target = cv2.GaussianBlur(target,(5,5),0)       #unsharpen all
        # target = cv2.medianBlur(target,33)              #remove cracks with with 33 pixel
        # target = cv2.bilateralFilter(target,33,90,90)   #bring back edges and remove textures
        # target = cv2.bilateralFilter(target,13,90,90)   #bring back edges and remove textures
        # target = cv2.medianBlur(target,53)              #remove big color patches
        # target = cv2.bilateralFilter(target,13,90,90)   #bring back edges and remove textures
tilman's avatar
tilman committed
116
117
        #target = cv2.medianBlur(target,11)              #smoothen cracks
        target = cv2.medianBlur(target,int(esz*5)+1 if int(esz*5)%2==0 else int(esz*5)) #smoothen cracks
tilman's avatar
tilman committed
118
119
        target = cv2.bilateralFilter(target,int(esz*25),int(esz*40),int(esz*40))        #remove cracks
        cv2.imwrite(os.path.basename(img_name)+'_inkm_step1_crackremoval.jpg',target)
tilman's avatar
tilman committed
120

tilman's avatar
tilman committed
121
122
123
        fposes = np.array([np.array([line[:2] for line in pose if line[2] > 0]) for pose in datum.poseKeypoints]) #filtered poses without zero lines
        mask = np.zeros((len(img),len(img[0]),1), np.uint8)
        kmout_mask = np.zeros((len(img),len(img[0]),1), np.uint8)
tilman's avatar
tilman committed
124
        for pose in fposes: #remove bodys
tilman's avatar
tilman committed
125
            convexhull = Polygon(pose).convex_hull
tilman's avatar
tilman committed
126
            #inpainting
tilman's avatar
tilman committed
127
            # sconvexhull = affinity.scale(convexhull, xfact=1.5, yfact=1.7, origin=convexhull.centroid)
128
            sconvexhull = affinity.scale(convexhull, xfact=1.7, yfact=1.4, origin=convexhull.centroid)
129
            cv2.drawContours(mask, [polyToArr(sconvexhull)], 0, 255, int(15*esz))
tilman's avatar
tilman committed
130
            cv2.drawContours(mask, [polyToArr(sconvexhull)], 0, 255, -1)
tilman's avatar
tilman committed
131
132

            #kmeans check
133
            sconvexhull = affinity.scale(convexhull, xfact=1, yfact=0.5, origin=convexhull.centroid)
134
            cv2.drawContours(kmout_mask, [polyToArr(sconvexhull)], 0, 255, int(7*esz))
tilman's avatar
tilman committed
135
            cv2.drawContours(kmout_mask, [polyToArr(sconvexhull)], 0, 255, -1)
tilman's avatar
tilman committed
136
        cv2.rectangle(mask, (0,0), (len(img[0]),len(img)), 255, int(40*esz)) #remove frames
tilman's avatar
tilman committed
137
138
        cv2.imwrite(os.path.basename(img_name)+'_inkm_step2_inpaintmask.jpg',mask)
        cv2.imwrite(os.path.basename(img_name)+'_inkm_step2_kmeansresmask.jpg',kmout_mask)
tilman's avatar
tilman committed
139
        inpainted = cv2.inpaint(target, mask, 3, cv2.INPAINT_TELEA)
tilman's avatar
tilman committed
140
141
        # inpainted = cv2.medianBlur(inpainted,int(esz*5)+1 if int(esz*5)%2==0 else int(esz*5))
        # inpainted = cv2.bilateralFilter(inpainted,30,40,40)
tilman's avatar
tilman committed
142
        cv2.imwrite(os.path.basename(img_name)+'_inkm_step3_inpainted.jpg',inpainted)
tilman's avatar
tilman committed
143
144
       
        #to kmeans
tilman's avatar
tilman committed
145
146
        kmeans_output = imgKmeans(inpainted, KMEANS_K)
        cv2.imwrite(os.path.basename(img_name)+'_inkm_step4_kmeansres.jpg',kmeans_output)
tilman's avatar
tilman committed
147

tilman's avatar
tilman committed
148
149
        # cv2.imshow(img_name+"inpainted", inpainted)
        # cv2.imshow(img_name+"mask", kmout_mask)
tilman's avatar
tilman committed
150
        kmout_mask=cv2.cvtColor(kmout_mask,cv2.COLOR_GRAY2BGR) #change mask to a 3 channel image 
tilman's avatar
tilman committed
151
        km_mask_out=cv2.subtract(kmout_mask,kmeans_output)        #subtract mask from kmeans result
tilman's avatar
tilman committed
152
153
154
        km_mask_out=cv2.subtract(kmout_mask,km_mask_out)

        #idea: count how many pixels the white mask has, count how many pixels each color from kmeans result appears in mask -> area > 80% is foreground
tilman's avatar
tilman committed
155
        kmeans_colors, kmeans_counts =   np.unique(kmeans_output.reshape(-1, kmeans_output.shape[-1]),axis=0,return_counts=True)
tilman's avatar
tilman committed
156
        km_mask_colors, km_mask_counts = np.unique(km_mask_out.reshape(-1, km_mask_out.shape[-1]),axis=0,return_counts=True)
tilman's avatar
tilman committed
157
158
        mask_colors, mask_counts =       np.unique(kmout_mask.reshape(-1, kmout_mask.shape[-1]),axis=0,return_counts=True)

159
160
161
        white_pixels = mask_counts[(np.argwhere(mask_colors>0))[0,0]]
        threshold_count = int(KMEANS_AREA_MASK_THRESHOLD*white_pixels) #select counts from white color and generate threshold with it.
        print("white_pixels",white_pixels,"threshold_count",threshold_count)
tilman's avatar
tilman committed
162
        
163
164
165
166
167
        kmeans_colors = kmeans_colors[kmeans_counts.argsort()][::-1] # sort kmeans_counts by count
        kmeans_counts = kmeans_counts[kmeans_counts.argsort()][::-1] # sort kmeans_counts by count
        
        km_mask_colors = km_mask_colors[km_mask_counts.argsort()][::-1] # sort mask colors by count
        km_mask_counts = km_mask_counts[km_mask_counts.argsort()][::-1] # sort mask counts by count
168
        print("km_mask_colors",km_mask_colors,"km_mask_counts",km_mask_counts)
tilman's avatar
tilman committed
169
170
        filtered_km_mask_FG_colors = km_mask_colors[np.argwhere(km_mask_counts>=threshold_count)]
        filtered_km_mask_FG_colors = filtered_km_mask_FG_colors[np.sum(filtered_km_mask_FG_colors, axis=2)>0] #filter out black from mask
tilman's avatar
tilman committed
171
        
tilman's avatar
tilman committed
172
173
        filtered_km_mask_BG_colors = km_mask_colors[np.argwhere(km_mask_counts<threshold_count)]
        filtered_km_mask_BG_colors = filtered_km_mask_BG_colors[np.sum(filtered_km_mask_BG_colors, axis=2)>0] #filter out black from mask
tilman's avatar
tilman committed
174
        
tilman's avatar
tilman committed
175
176
177
178
179
180
181
182
183
184

        if(len(filtered_km_mask_FG_colors)>0): #we can not do anything if we have no foreground colors

            cv2.putText(km_mask_out, "Foreground colors:", (10,25), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255,255,255))
            
            cv2.rectangle(kmeans_output, (0,0), (235,30), (0,0,0), -1)
            cv2.putText(kmeans_output, "Foreground color:", (10,25), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255,255,255))
            cv2.rectangle(kmeans_output, (220,15), (230,25), (int(filtered_km_mask_FG_colors[0][0]),int(filtered_km_mask_FG_colors[0][1]),int(filtered_km_mask_FG_colors[0][2])), -1)
            cv2.rectangle(kmeans_output, (220,15), (230,25), (255,255,255), 1)

tilman's avatar
tilman committed
185
186
187
188
            cv2.rectangle(km_mask_out, (0,0), (235,30), (0,0,0), -1)
            cv2.putText(km_mask_out, "Foreground colors:", (10,25), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255,255,255))
            cv2.rectangle(km_mask_out, (220,15), (230,25), (255,255,255), 1)

tilman's avatar
tilman committed
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
            # cv2.imwrite(os.path.basename(img_bdcn)+'_km_mask_out_before.jpg',kmeans_output)
            binary_output = kmeans_output.copy()
            offset=220;
            for color in filtered_km_mask_FG_colors:
                #print colors on screen
                print("filtered_km_mask_FG_colors",color)
                x = offset
                y = 15
                offset += 15
                #replace colors
                # binary_output[((binary_output[:,:,0] == color[0]) & (binary_output[:,:,1] == color[1]) & (binary_output[:,:,2] == color[2]))]=filtered_km_mask_FG_colors[0] #search in kmeans image for specific color and replace all by first color
                binary_output[((binary_output[:,:,0] == color[0]) & (binary_output[:,:,1] == color[1]) & (binary_output[:,:,2] == color[2]))]=[255,255,255] #search in binary_output image for specific color and replace all by first color
                kmeans_output[((kmeans_output[:,:,0] == color[0]) & (kmeans_output[:,:,1] == color[1]) & (kmeans_output[:,:,2] == color[2]))]=filtered_km_mask_FG_colors[0] #search in kmeans_output image for specific color and replace all by first color
                # km_mask_out[((km_mask_out[:,:,0] == color[0]) & (km_mask_out[:,:,1] == color[1]) & (km_mask_out[:,:,2] == color[2]))]=filtered_km_mask_FG_colors[0] #search in binary_output image for specific color and replace all by first color
                cv2.rectangle(km_mask_out, (x,y), (x+10,y+10), (int(color[0]),int(color[1]),int(color[2])), -1)
            for color in filtered_km_mask_BG_colors:
                print("filtered_km_mask_BG_colors",color)
                binary_output[((binary_output[:,:,0] == color[0]) & (binary_output[:,:,1] == color[1]) & (binary_output[:,:,2] == color[2]))]=[0,0,0] #search in binary_output image for specific color and replace all by first color
                # binary_output[((binary_output[:,:,0] == color[0]) & (binary_output[:,:,1] == color[1]) & (binary_output[:,:,2] == color[2]))]=filtered_km_mask_BG_colors[0] #search in binary_output image for specific color and replace all by first color
                # km_mask_out[((km_mask_out[:,:,0] == color[0]) & (km_mask_out[:,:,1] == color[1]) & (km_mask_out[:,:,2] == color[2]))]=filtered_km_mask_BG_colors[0] #search in binary_output image for specific color and replace all by first color
            colors_only_BG = kmeans_colors[np.invert(np.isin(kmeans_colors, km_mask_colors).all(axis=1))]    #colors in kmean_out wich do not appear in any mask
            for color in colors_only_BG: 
                print("colors_only_BG",color)
                binary_output[((binary_output[:,:,0] == color[0]) & (binary_output[:,:,1] == color[1]) & (binary_output[:,:,2] == color[2]))]=[0,0,0] #search in binary_output image for specific color and replace all by first color
                # kmeans_output[((kmeans_output[:,:,0] == color[0]) & (kmeans_output[:,:,1] == color[1]) & (kmeans_output[:,:,2] == color[2]))]=colors_only_BG[0] #search in kmeans_output image for specific color and replace all by first color
                # km_mask_out[((km_mask_out[:,:,0] == color[0]) & (km_mask_out[:,:,1] == color[1]) & (km_mask_out[:,:,2] == color[2]))]=colors_only_BG[0] #search in binary_output image for specific color and replace all by first color

tilman's avatar
tilman committed
216
            cv2.imwrite(os.path.basename(img_name)+'_inkm_step5_kmeans_masked.jpg',km_mask_out)
tilman's avatar
tilman committed
217
218
219
220
            cv2.imwrite(os.path.basename(img_name)+'_inkm_step5_kmean_colorreplaced.jpg',kmeans_output)
            cv2.imwrite(os.path.basename(img_name)+'_inkm_step5_kmean_binarization.jpg',binary_output)
            

tilman's avatar
tilman committed
221

tilman's avatar
tilman committed
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
            # cv2.imwrite(os.path.basename(img_bdcn)+'_kmeans_output_after.jpg',kmeans_output)
            # cv2.imwrite(os.path.basename(img_bdcn)+'_km_mask_out.jpg',km_mask_out)
            # cv2.namedWindow("binary_output", cv2.WINDOW_NORMAL)

            # cv2.imshow("binary_output", binary_output)
            # cv2.waitKey(0)
            #apply erosure/dilation morphing filters
            # kernel = np.ones((5,5),np.uint8)
            
            #reprint text because medianBlur destroys it
            # cv2.imshow("binary_output", binary_output)
            # cv2.waitKey(0)


            if COLORED_CANVAS:
                #remove small kmeans fragments
                kmeans_output = cv2.medianBlur(kmeans_output,7)
                cv2.rectangle(kmeans_output, (0,0), (235,30), (0,0,0), -1)
                cv2.putText(kmeans_output, "Foreground color:", (10,25), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255,255,255))
                cv2.rectangle(kmeans_output, (220,15), (230,25), (int(filtered_km_mask_FG_colors[0][0]),int(filtered_km_mask_FG_colors[0][1]),int(filtered_km_mask_FG_colors[0][2])), -1)
                cv2.rectangle(kmeans_output, (220,15), (230,25), (255,255,255), 1)
                
                cv2.imwrite(os.path.basename(img_name)+'_inkm_step6_kmean_fragementsremoved.jpg',kmeans_output)
                output_canvas = kmeans_output
            else:
                # try:
                #     #medianblur randomly crashes. we can also continue without
                #     binary_output = cv2.medianBlur(binary_output,int(esz*60)+1 if int(esz*60)%2==0 else int(esz*60))
                # finally:
                #     pass
                
                kernel = np.array([
                    [0,0,1,0,0],
                    [0,1,1,1,0],
                    [0.5,1,1,1,1],
                    [0,1,1,1,0],
                    [0,0,0.5,0,0]], dtype=np.uint8)

                binary_output = cv2.dilate(binary_output,kernel,iterations = 10)
                binary_output = cv2.erode(binary_output,kernel,iterations = 10)
                # cv2.imshow("binary_output", binary_output)
                # cv2.waitKey(0)
                # cv2.imshow("binary_output", binary_output)
                # cv2.waitKey(0)
                cv2.imwrite(os.path.basename(img_name)+'_inkm_step6_1_kmean_morphclose.jpg',binary_output)
                # binary_output = cv2.medianBlur(binary_output,int(esz*60)+1 if int(esz*60)%2==0 else int(esz*60))
                # cv2.imwrite(os.path.basename(img_name)+'_inkm_step6_2_kmean_morpblurred.jpg',binary_output)
                # cv2.imshow("binary_output", binary_output)
                # cv2.waitKey(0)
                binary_output = cv2.erode(binary_output,kernel,iterations = 10)
                binary_output = cv2.dilate(binary_output,kernel,iterations = 10)
                # cv2.imshow("binary_output", binary_output)
                # cv2.waitKey(0)
                cv2.imwrite(os.path.basename(img_name)+'_inkm_step6_2_kmean_morpopen.jpg',binary_output)
                # cv2.imshow("binary_output", binary_output)
                # cv2.waitKey(0)

                # binary_output = 
                # cv2.imshow(img_name+"kmeans_masked", km_mask_out)
                #replace colors
                # binary_output = cv2.medianBlur(binary_output,53)
                # binary_output = cv2.bilateralFilter(binary_output,30,60,60)
                # kmeans_bgfg = imgKmeans(binary_output, 3)
                # cv2.namedWindow("kmeans_bgfg", cv2.WINDOW_NORMAL)
                # cv2.imshow("kmeans_bgfg", kmeans_bgfg)
                cv2.rectangle(binary_output, (0,0), (235,30), (0,0,0), -1)
                cv2.putText(binary_output, "Foreground color:", (10,25), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255,255,255))
                cv2.rectangle(binary_output, (220,15), (230,25), (int(filtered_km_mask_FG_colors[0][0]),int(filtered_km_mask_FG_colors[0][1]),int(filtered_km_mask_FG_colors[0][2])), -1)
                cv2.rectangle(binary_output, (220,15), (230,25), (255,255,255), 1)
                output_canvas = binary_output
        else:
            print("WARNING: skipped fg/bg calc")
tilman's avatar
tilman committed
294
295


tilman's avatar
tilman committed
296
    if BDCN_EDGES:
tilman's avatar
tilman committed
297
298
        #overlay = np.array([[[255,255,255]]*len(img[0])]*len(img),np.uint8) #white image canvas
        #output_canvas = overlay_two_image_v2(output_canvas, overlay, [0,0,0], 0.2)
tilman's avatar
tilman committed
299
300
301
302
303
304
305
306
        fposes = np.array([np.array([line[:2] for line in pose if line[2] > 0]) for pose in datum.poseKeypoints]) #filtered poses without zero lines
        for pose in fposes:
            convexhull = Polygon(pose).convex_hull
            sconvexhull = affinity.scale(convexhull, xfact=1.6, yfact=1.6, origin=convexhull.centroid)
            # print(convexhull)
            #cv2.line(img, (int(pose[0][0]),int(pose[0][1])), (int(pose[1][0]),int(pose[1][1])), (0,255,0), int(6*esz))
            # cv2.drawContours(img, [polyToArr(sconvexhull)], 0, 255, int(10*esz))
            # cv2.drawContours(img, [polyToArr(sconvexhull)], 0, 255, -1)
tilman's avatar
tilman committed
307
308
            cv2.drawContours(bdcn_image, [polyToArr(sconvexhull)], 0, (255,255,255), int(10*esz))
            cv2.drawContours(bdcn_image, [polyToArr(sconvexhull)], 0, (255,255,255), -1)
tilman's avatar
tilman committed
309
310
        #draw rectangle
        # cv2.rectangle(img, (0,0), (len(img[0]),len(img)), (255,0,0), int(40*esz))
tilman's avatar
tilman committed
311
        cv2.rectangle(bdcn_image, (0,0), (len(img[0]),len(img)), (255,255,255), int(40*esz)) #eliminate rectangle from frames
312

tilman's avatar
tilman committed
313
        blurred = cv2.medianBlur(bdcn_image,int(esz*5)+1 if int(esz*5)%2==0 else int(esz*5))
tilman's avatar
tilman committed
314
        print("med1")
tilman's avatar
tilman committed
315
        blurred = cv2.bilateralFilter(blurred,30,60,60)
tilman's avatar
tilman committed
316
        print("bil1")
tilman's avatar
tilman committed
317
        blurred = cv2.bilateralFilter(blurred,30,60,60)
tilman's avatar
tilman committed
318
        print("bil2")
tilman's avatar
tilman committed
319
        blurred = cv2.bilateralFilter(blurred,30,60,60)
tilman's avatar
tilman committed
320
        print("bil3")
tilman's avatar
tilman committed
321
        blurred = cv2.bilateralFilter(blurred,30,60,60)
tilman's avatar
tilman committed
322
        print("bil4")
tilman's avatar
tilman committed
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
        #output_canvas = overlay_two_image_v2(output_canvas, blurred, [0,0,0], 0.5)

        #output_canvas = cv2.addWeighted(blurred, 0.2, output_canvas, 0.8, 0.0)

        # cv2.imshow(img_name+"blurred", blurred)
        rows,cols,channels = blurred.shape
        roi = output_canvas[0:rows, 0:cols ]
        img2gray = cv2.cvtColor(blurred,cv2.COLOR_BGR2GRAY)
        ret, mask = cv2.threshold(img2gray, 100, 255, cv2.THRESH_BINARY_INV)
        mask_inv = cv2.bitwise_not(mask)
        # cv2.imshow(img_name+"mask_inv", mask_inv)
        output_canvas = cv2.bitwise_and(roi,roi,mask = mask_inv)

        

tilman's avatar
tilman committed
338

339
340
341
342
343
344
345
346
347
348
349
    if TRIANGLES or BODY_LINES:
        triangles = [poseToTriangle(pose) for pose in datum.poseKeypoints]
        for triangle in triangles:
            if triangle is not None:
                if TRIANGLES:
                    cv2.drawContours(img, [triangle], 0, 255, -1)
                    cv2.drawContours(output_canvas, [triangle], 0, 255, -1)
                if BODY_LINES:
                    linePoints = triangleToBodyLine(triangle)
                    cv2.line(img, trp(linePoints[0]), trp(linePoints[1]), (0,255,0), int(6*esz))
                    cv2.line(output_canvas, trp(linePoints[0]), trp(linePoints[1]), (0,255,0), int(6*esz))
tilman's avatar
tilman committed
350

351
352
    if BISEC_CONES or GLOBAL_LINE:
        bisecCones = [poseToBisectCone(pose, max_lw, BISEC_CONE_ANGLE, CORRECTION_ANGLE) for pose in datum.poseKeypoints] #use max(img.height,width) as cone length
tilman's avatar
tilman committed
353
354
        bisecCones = [v for v in bisecCones if v] #remove None values
        #print([polyToArr(bisecCone) for bisecCone in bisecCones])
tilman's avatar
tilman committed
355
        intersections = coneIntersections(bisecCones)
tilman's avatar
tilman committed
356
        maxlevel = max(map(lambda t: len(t), intersections.keys()))
tilman's avatar
tilman committed
357
        for combi in intersections:
tilman's avatar
tilman committed
358
359
360
361
362
            is_not_last_level = len(combi) < maxlevel
            if is_not_last_level and BISEC_SKIP_LOWER_LEVELS:
                continue;
            else:
                overlay = np.zeros((len(img),len(img[0]),3), np.uint8)
363
364
365
366
367
                color = min(((len(combi)-1)*100,255))
                alpha = OVERLAY_ALPHA
                if DRAW_FIRST_CONE_LAYER_BRIGTHER and len(combi) == 1:
                    cv2.drawContours(overlay, [polyToArr(intersections[combi])], 0, (0,255,0), -1)
                    img = overlay_two_image_v2(img, overlay, [0,0,0], (0.25))
368
369
                if BISEC_CONES:
                    cv2.drawContours(overlay, [polyToArr(intersections[combi])], 0, (color,0,(0 if is_not_last_level else 255)), -1)
370
                img = overlay_two_image_v2(img, overlay, [0,0,0], (alpha if is_not_last_level else 0.6))
tilman's avatar
tilman committed
371
372
373
                if not is_not_last_level and GLOBAL_LINE: #draw centroid of last polygon
                    xy = (int(intersections[combi].centroid.x),int(intersections[combi].centroid.y))
                    global_angle = getGlobalLineAngle(datum.poseKeypoints, CORRECTION_ANGLE)
374
375
                    print("global_angle",np.rad2deg(global_angle))
                    dist = max_lw
tilman's avatar
tilman committed
376
                    d = (int(dist * np.cos(global_angle)), int(dist * np.sin(global_angle))) #with origin zero
377
378
379
                    d_l = (int(-dist * np.cos(global_angle)), int(-dist * np.sin(global_angle))) #with origin zero
                    # draw line with global gaze angle (special mean of all gaze angles) and through center of last intersection
                    cv2.line(img, xy, (xy[0]+d[0],xy[1]-d[1]), (0,255,255), int(10*esz))
380
                    cv2.line(output_canvas, xy, (xy[0]+d[0],xy[1]-d[1]), (0,255,255), int(10*esz))
381
                    cv2.line(img, xy, (xy[0]+d_l[0],xy[1]-d_l[1]), (0,255,255), int(10*esz))
382
                    cv2.line(output_canvas, xy, (xy[0]+d_l[0],xy[1]-d_l[1]), (0,255,255), int(10*esz))
tilman's avatar
tilman committed
383
                    cv2.circle(img, xy, int(13*esz), (255,255,0), -1)
384
                    cv2.circle(output_canvas, xy, int(13*esz), (255,255,0), -1)
385
386
387
388
389
390

    if BISEC_VECTORS:
        bisecVectors = [poseToBisectVector(pose, CORRECTION_ANGLE) for pose in datum.poseKeypoints]
        for bisecVector in bisecVectors:
            if bisecVector is not None:
                cv2.arrowedLine(img, trp(bisecVector[1]), trp(bisecVector[0]), (0,0,255), int(4*esz))
391
                cv2.arrowedLine(output_canvas, trp(bisecVector[1]), trp(bisecVector[0]), (0,0,255), int(4*esz))
tilman's avatar
tilman committed
392

tilman's avatar
tilman committed
393
    if SAVE_FILE:
394
        #cv2.imwrite(os.path.basename(img_name),img)
tilman's avatar
tilman committed
395
396
        if COLORED_CANVAS:
            cv2.imwrite(os.path.basename(img_bdcn)+'_final_colored_canvas.jpg',output_canvas)
397
            print("saved _final_colored_canvas")
tilman's avatar
tilman committed
398
399
        else:
            cv2.imwrite(os.path.basename(img_bdcn)+'_final_binary_canvas.jpg',output_canvas)
tilman's avatar
tilman committed
400
401
402
    if SHOW_WINDOW:
        cv2.namedWindow(img_name, cv2.WINDOW_NORMAL)
        cv2.imshow(img_name, img)
tilman's avatar
tilman committed
403
404
        cv2.namedWindow(img_bdcn+"canvas", cv2.WINDOW_NORMAL)
        cv2.imshow(img_bdcn+"canvas", output_canvas)
tilman's avatar
tilman committed
405
        cv2.waitKey(0)
tilman's avatar
tilman committed
406
if SHOW_WINDOW:
407
    cv2.waitKey(0)
tilman's avatar
tilman committed
408
    cv2.destroyAllWindows()