IDS/IDS_detection/IDS_freeze_trigger.py

from ctypes import *
from pyueye import ueye
import numpy as np
import cv2
import sys
import ctypes
import struct
import threading
import time
import datetime
import os
import pymysql
import tensorflow as tf
import requests as req
import sys
import mysql.connector
from mysql.connector import Error
from urllib import parse
from PIL import Image



#setting
#-------------------------------------------------------------------------------
hCam = ueye.HIDS(0)
sensor_info = ueye.SENSORINFO()
camera_info = ueye.CAMINFO()
pcImageMemory = ueye.c_mem_p()
MemID = ueye.int()
rectAOI = ueye.IS_RECT()
rectAOI.s32X     = 100
rectAOI.s32Y     = 100
rectAOI.s32Width = 800
rectAOI.s32Height = 600
pitch = ueye.INT()
nBitsPerPixel = ueye.INT(24)    #24: bits per pixel for color mode; take 8 bits per pixel for monochrome
channels = 3                    #3: channels for color mode(RGB); take 1 channel for monochrome
m_nColorMode = ueye.INT()		# Y8/RGB16/RGB24/REG32
bytes_per_pixel = int(nBitsPerPixel / 8)
dZoomValue = ueye.DOUBLE()
#fps = ueye.DOUBLE(60) # set you want fps
Real_FPS =ueye.DOUBLE() # get real fps
exposure_value = ueye.DOUBLE(10) # set exposure value
cbSizeOfParam = 8
BurstSize = ueye.UINT(1)
status_new=0
status_old=0
trigger_delay_time = ueye.UINT(0)
#-------------------------------------------------------------------------------


print("START")
# Starts the driver and establishes the connection to the camera
nRet = ueye.is_InitCamera(hCam, None)
if nRet != ueye.IS_SUCCESS:
    ueye.is_InitCamera(hCam, None)
    #print("is_InitCamera ERROR")

nRet = ueye.is_SetExternalTrigger(hCam, ueye.IS_SET_TRIGGER_HI_LO)
if nRet != ueye.IS_SUCCESS:
    print("is_SetExternalTrigger ERROR")

nRet = ueye.is_SetTriggerDelay(hCam,trigger_delay_time)
if nRet != ueye.IS_SUCCESS:
    print("is_SetTriggerDelay ERROR")
print(nRet)

nRet = ueye.is_Trigger(hCam, ueye.IS_TRIGGER_CMD_SET_BURST_SIZE,BurstSize,ueye.sizeof(BurstSize))
if nRet != ueye.IS_SUCCESS:
    print("is_Trigger ERROR")

# Reads out the data hard-coded in the non-volatile camera memory and writes it to the data structure that cInfo points to
nRet = ueye.is_GetCameraInfo(hCam, camera_info)
if nRet != ueye.IS_SUCCESS:
    print("is_GetCameraInfo ERROR")

# You can query additional information about the sensor type used in the camera
nRet = ueye.is_GetSensorInfo(hCam, sensor_info)
if nRet != ueye.IS_SUCCESS:
    print("is_GetSensorInfo ERROR")
'''
nRet = ueye.is_ResetToDefault(hCam)
if nRet != ueye.IS_SUCCESS:
    print("is_ResetToDefault ERROR")
'''
# Set display mode to DIB
nRet = ueye.is_SetDisplayMode(hCam, ueye.IS_SET_DM_DIB)

# Set the right color mode
if int.from_bytes(sensor_info.nColorMode.value, byteorder='big') == ueye.IS_COLORMODE_BAYER:
    # setup the color depth to the current windows setting
    ueye.is_GetColorDepth(hCam, nBitsPerPixel, m_nColorMode)
    bytes_per_pixel = int(nBitsPerPixel / 8)
    print("IS_COLORMODE_BAYER: ", )
    print("\tm_nColorMode:", m_nColorMode)
    print("\tnBitsPerPixel:", nBitsPerPixel)
    print("\tbytes_per_pixel:", bytes_per_pixel)

elif int.from_bytes(sensor_info.nColorMode.value, byteorder='big') == ueye.IS_COLORMODE_CBYCRY:
    # for color camera models use RGB32 mode
    m_nColorMode = ueye.IS_CM_BGRA8_PACKED
    nBitsPerPixel = ueye.INT(32)
    bytes_per_pixel = int(nBitsPerPixel / 8)
    print("IS_COLORMODE_CBYCRY: ", )
    print("\tm_nColorMode: \t\t", m_nColorMode)
    print("\tnBitsPerPixel: \t\t", nBitsPerPixel)
    print("\tbytes_per_pixel: \t\t", bytes_per_pixel)

elif int.from_bytes(sensor_info.nColorMode.value, byteorder='big') == ueye.IS_COLORMODE_MONOCHROME:
    # for color camera models use RGB32 mode
    m_nColorMode = ueye.IS_CM_MONO8
    nBitsPerPixel = ueye.INT(8)
    bytes_per_pixel = int(nBitsPerPixel / 8)
    print("IS_COLORMODE_MONOCHROME: ", )
    print("\tm_nColorMode: \t\t", m_nColorMode)
    print("\tnBitsPerPixel: \t\t", nBitsPerPixel)
    print("\tbytes_per_pixel: \t\t", bytes_per_pixel)
else:
    # for monochrome camera models use Y8 mode
    m_nColorMode = ueye.IS_CM_MONO8
    nBitsPerPixel = ueye.INT(8)
    bytes_per_pixel = int(nBitsPerPixel / 8)
    print("else")

'''
nRet = ueye.is_AOI(hCam, ueye.IS_AOI_IMAGE_SET_AOI, rectAOI, ueye.sizeof(rectAOI))
if nRet != ueye.IS_SUCCESS:
    print("is_AOI ERROR")
'''
# Can be used to set the size and position of an "area of interest"(AOI) within an image
nRet = ueye.is_AOI(hCam, ueye.IS_AOI_IMAGE_GET_AOI, rectAOI, ueye.sizeof(rectAOI))
if nRet != ueye.IS_SUCCESS:
    print("is_AOI ERROR")
width = rectAOI.s32Width
height = rectAOI.s32Height

print("Exposure value : {}".format(exposure_value))
print("Maximum image width:", width)
print("Maximum image height:", height)
# ---------------------------------------------------------------------------------------------------------------------------------------
# Allocates an image memory for an image having its dimensions defined by width and height and its color depth defined by nBitsPerPixel
nRet = ueye.is_AllocImageMem(hCam, width, height, nBitsPerPixel, pcImageMemory, MemID)
if nRet != ueye.IS_SUCCESS:
    print("is_AllocImageMem ERROR")
else:
    # Makes the specified image memory the active memory
    nRet = ueye.is_SetImageMem(hCam, pcImageMemory, MemID)
    if nRet != ueye.IS_SUCCESS:
        print("is_SetImageMem ERROR")
    else:
        # Set the desired color mode
        nRet = ueye.is_SetColorMode(hCam, m_nColorMode)

# Activates the camera's live video mode (free run mode)
nRet = ueye.is_FreezeVideo(hCam, ueye.IS_DONT_WAIT)
if nRet != ueye.IS_SUCCESS:
    print("is_FreezeVideo ERROR")

# Enables the queue mode for existing image memory sequences
nRet = ueye.is_InquireImageMem(hCam, pcImageMemory, MemID, width, height, nBitsPerPixel, pitch)
if nRet != ueye.IS_SUCCESS:
    print("is_InquireImageMem ERROR")
else:
    print("Press q to leave the programm")
# ---------------------------------------------------------------------------------------------------------------------------------------
Count=0


def cut_rectangle():
    image_size = 150
    # img = cv2.imread("D:\\fatwolf\\company_files\\opencv\\2021-05-05-11_13_47.png")
    # img = cv2.imread("D:\\fatwolf\\company_files\\opencv\\2.png")
    #img = cv2.imread("C:\\Users\\User\\Desktop\\tfcoffebean\\test\\1.png")
    # img = cv2.imread("C:\\Users\\User\\Desktop\\IDS\\p\\12033_248.png")
    # img_size = img.shape
    # print(img_size)
    img = cv2.imread('D:\\fatwolf\\company_files\\python_code\\test_code\\test_pic\\12033_267.png')

    # img = cv2.resize(img1,(968,548))
    point_color = (0, 0, 255)
    command1 = "SELECT Name,X, X1 ,Y ,Y1 FROM `cut` WHERE Name LIKE 'roi1'"
    l = conn.cursor()
    l.execute(command1)
    conn.commit()
    r1 = l.fetchone()

    # print(r1[0])
    count = 1
    def roi1():
        # x = r1[1]
        # x1 = r1[2]
        # y = r1[3]
        # y1 = r1[4]
        #x = 743
        #x1 = 892
        #y = 17
        #y1 = 164
        x = 1257
        x1 = 1355
        y = 185
        y1 = 278
        i = 1
        i1 = 1
        i2 = 1
        i3 = 1
        i4 = 1
        i5 = 1
        i6 = 1
        number = count

        for i in range(6):
            roi = img[y:y1, x:x1]
            cv2.rectangle(img, (x, y), (x1, y1), point_color, 1)
            roi = cv2.resize(roi, (image_size, image_size), 0, 0, cv2.INTER_LINEAR)
            cv2.imwrite('D:\\fatwolf\\company_files\\python_code\\test_code\\test_pic\\pic' + '00_' + str(number) + '.png', roi)
            number = number + 1
            y = y + 150
            y1 = y1 + 150
        x = x + 145
        x1 = x1 + 145
        y = 1355
        y1 = 278
        '''
        for i in range(6):
            roi = img[y:y1, x:x1]
            cv2.rectangle(img, (x, y), (x1, y1), point_color, 1)
            roi = cv2.resize(roi, (image_size, image_size), 0, 0, cv2.INTER_LINEAR)
            cv2.imwrite('D:\\fatwolf\\company_files\\paper_coffee\\pic\\' + '00_' + str(number) + '.png', roi)
            number = number + 1
            x = x + 150
            x1 = x1 + 150
        y = y + 145
        y1 = y1 + 145
        x = 1257
        x1 = 1355
        '''
        '''
        for i1 in range(6):
            roi = img[y:y1, x:x1]
            cv2.rectangle(img, (x, y), (x1, y1), point_color, 1)
            roi = cv2.resize(roi, (image_size, image_size), 0, 0, cv2.INTER_LINEAR)
            cv2.imwrite('D:\\fatwolf\\company_files\\paper_coffee\\pic\\' + '00_' + str(number) + '.png', roi)
            number = number + 1
            x = x + 150
            x1 = x1 + 150
        y = y + 145
        y1 = y1 + 145
        x = 743
        x1 = 892

        for i2 in range(6):
            roi = img[y:y1, x:x1]
            cv2.rectangle(img, (x, y), (x1, y1), point_color, 1)
            roi = cv2.resize(roi, (image_size, image_size), 0, 0, cv2.INTER_LINEAR)
            cv2.imwrite('D:\\fatwolf\\company_files\\paper_coffee\\pic\\' + '00_' + str(number) + '.png', roi)
            number = number + 1
            x = x + 150
            x1 = x1 + 150
        y = y + 145
        y1 = y1 + 145
        x = 743
        x1 = 892

        for i3 in range(6):
            roi = img[y:y1, x:x1]
            cv2.rectangle(img, (x, y), (x1, y1), point_color, 1)
            roi = cv2.resize(roi, (image_size, image_size), 0, 0, cv2.INTER_LINEAR)
            cv2.imwrite('D:\\fatwolf\\company_files\\paper_coffee\\pic\\' + '00_' + str(number) + '.png', roi)
            number = number + 1
            x = x + 150
            x1 = x1 + 150
        y = y + 145
        y1 = y1 + 145
        x = 743
        x1 = 892

        for i4 in range(6):
            roi = img[y:y1, x:x1]
            cv2.rectangle(img, (x, y), (x1, y1), point_color, 1)
            roi = cv2.resize(roi, (image_size, image_size), 0, 0, cv2.INTER_LINEAR)
            cv2.imwrite('D:\\fatwolf\\company_files\\paper_coffee\\pic\\' + '00_' + str(number) + '.png', roi)
            number = number + 1
            x = x + 150
            x1 = x1 + 150
        y = y + 145
        y1 = y1 + 145
        x = 743
        x1 = 892

        for i5 in range(6):
            roi = img[y:y1, x:x1]
            cv2.rectangle(img, (x, y), (x1, y1), point_color, 1)
            roi = cv2.resize(roi, (image_size, image_size), 0, 0, cv2.INTER_LINEAR)
            cv2.imwrite('D:\\fatwolf\\company_files\\paper_coffee\\pic\\' + '00_' + str(number) + '.png', roi)
            number = number + 1
            x = x + 150
            x1 = x1 + 150
        y = y + 145
        y1 = y1 + 145
        x = 743
        x1 = 892

        for i6 in range(6):
            roi = img[y:y1, x:x1]
            cv2.rectangle(img, (x, y), (x1, y1), point_color, 1)
            roi = cv2.resize(roi, (image_size, image_size), 0, 0, cv2.INTER_LINEAR)
            cv2.imwrite('D:\\fatwolf\\company_files\\paper_coffee\\pic\\' + '00_' + str(number) + '.png', roi)
            number = number + 1
            x = x + 150
            x1 = x1 + 150
        y = y + 145
        y1 = y1 + 145
        x = 743
        x1 = 892
    '''
    start = datetime.datetime.now()
    roi1()
    end = datetime.datetime.now()
    print("cut_rectangle Run Time：", end - start)


def cnn():
    # data file

    data_dir = r"D:\\fatwolf\\company_files\\python_code\\test_code\\test_pic\\pic"

    print(data_dir)
    allName = os.listdir(data_dir)

    # train or test
    train = False
    # model address
    model_path = "model/image_model"
    allTestDataName = []

    def read_data(data_dir):
        datas = []
        labels = []
        fpaths = []
        for filename in os.listdir(data_dir):
            fpath = os.path.join(data_dir, filename)
            allTestDataName.append(filename)
            image = Image.open(fpath)
            data = np.array(image) / 255.0
            label = int(filename.split("_")[0])
            datas.append(data)
            labels.append(label)
            # allTestDataName.append(filename)

        datas = np.array(datas)
        labels = np.array(labels)
        allTestDataName.sort(key=lambda x: int(x[:-4]))
        # print(allTestDataName)

        # print("shape of datas: {}\tshape of labels: {}".format(datas.shape, labels.shape))
        return allTestDataName, datas, labels

    allTestDataName, datas, labels = read_data(data_dir)
    # num_classes = len(set(labels))
    num_classes = 4

    datas_placeholder = tf.compat.v1.placeholder(tf.float32, [None, 150, 150, 3])

    labels_placeholder = tf.compat.v1.placeholder(tf.int32, [None])

    dropout_placeholdr = tf.compat.v1.placeholder(tf.float32)

    conv0 = tf.layers.conv2d(datas_placeholder, 20, 5, activation=tf.nn.relu)
    pool0 = tf.layers.max_pooling2d(conv0, [2, 2], [2, 2])

    conv1 = tf.layers.conv2d(pool0, 40, 4, activation=tf.nn.relu)
    pool1 = tf.layers.max_pooling2d(conv1, [2, 2], [2, 2])

    flatten = tf.layers.flatten(pool1)

    fc = tf.layers.dense(flatten, 400, activation=tf.nn.relu)

    dropout_fc = tf.layers.dropout(fc, dropout_placeholdr)

    logits = tf.layers.dense(dropout_fc, num_classes)

    predicted_labels = tf.arg_max(logits, 1)

    losses = tf.nn.softmax_cross_entropy_with_logits(
        labels=tf.one_hot(labels_placeholder, num_classes),
        logits=logits
    )
    mean_loss = tf.reduce_mean(losses)

    optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=1e-2).minimize(losses)

    saver = tf.compat.v1.train.Saver()

    with tf.compat.v1.Session() as sess:

        if train:
            print("train mode")

            sess.run(tf.global_variables_initializer())

            train_feed_dict = {
                datas_placeholder: datas,
                labels_placeholder: labels,
                dropout_placeholdr: 0.25
            }
            for step in range(500):
                _, mean_loss_val = sess.run([optimizer, mean_loss], feed_dict=train_feed_dict)

                if step % 50 == 0:
                    print("step = {}\tmean loss = {}".format(step, mean_loss_val))
            saver.save(sess, model_path)
            print("train done save model{}".format(model_path))
        else:
            # print("reloading model")
            saver.restore(sess, model_path)
            # print("{}reload model".format(model_path))

            label_name_dict = {
                0: "Brokenbeans",
                1: "Peaberry",
                2: "shellbean",
                3: "Worms"
            }
            test_feed_dict = {
                datas_placeholder: datas,
                labels_placeholder: labels,
                dropout_placeholdr: 0
            }
            predicted_labels_val = sess.run(predicted_labels, feed_dict=test_feed_dict)

            for fpath, real_label, predicted_label in zip(allTestDataName, labels, predicted_labels_val):
                real_label_name = label_name_dict[real_label]
                # print("訓練前",real_label_name)
                predicted_label_name = label_name_dict[predicted_label]
                # print("訓練後",predicted_label_name)
                # print("{}\t => {}".format(fpath, predicted_label_name))
                fpath = os.path.basename(fpath)

                print(f"{fpath}\t => {predicted_label_name}")
                path1 = 'output.txt'
                f = open(path1, 'a+')
                f.write(f"{fpath} => {predicted_label_name}""\n")
                f.close()
                if predicted_label_name == "shellbean" or "Peaberry" or "Worms":
                    print("觸發噴嘴")
                else:
                    print('沒有觸發')
                '''
                sqlStuff = "INSERT INTO result(picname,identify)""VALUES (%s,%s)"
                data = [(fpath, predicted_label_name)]
                a = conn.cursor()
                a.executemany(sqlStuff, data)
                conn.commit()

                try:
                    connection = mysql.connector.connect(
                        host='127.0.0.1',  # 主機名稱
                        database='coffee_detection',  # 資料庫名稱
                        user='root',  # 帳號
                        password='g53743001')  # 密碼
                    # 新增資料
                    sql = "INSERT INTO result(picname,identify)""VALUES (%s,%s)"
                    new_data = ("test", "test2")
                    cursor = connection.cursor()
                    cursor.execute(sql, new_data)

                    # 確認資料有存入資料庫
                    connection.commit()

                except Error as e:
                    print("資料庫連接失敗：", e)

                finally:
                    if (connection.is_connected()):
                        cursor.close()
                        connection.close()

                '''
            dirListing = os.listdir(data_dir)
            # print(len(dirListing))







# Continuous image display
while (nRet == ueye.IS_SUCCESS):
    # In order to display the image in an OpenCV window we need to...
    # ...extract the data of our image memory
    array = ueye.get_data(pcImageMemory, width, height, nBitsPerPixel, pitch, copy=False)
    frame = np.reshape(array, (height.value, width.value, bytes_per_pixel))
    #frame = cv2.resize(frame, (1280,1024), fx=0.5, fy=0.5)
    cv2.imshow("SimpleLive_Python_uEye_OpenCV", frame)
    trigger_status = ueye.is_SetExternalTrigger(hCam,ueye.IS_GET_TRIGGER_STATUS)
    status_new = trigger_status
    print(status_old, status_new)

    if status_old == 1 and status_new == 0:
        #for i in range(1):
        now = time.strftime("%Y-%m-%d-%H_%M_%S", time.localtime(time.time()))
        FileParams = ueye.IMAGE_FILE_PARAMS()
        FileParams.pwchFileName = "C:/Users/User/Desktop/IDS/test/status04/" + now + ".png"
        FileParams.nFileType = ueye.IS_IMG_PNG
        FileParams.ppcImageMem = None
        FileParams.pnImageID = None
        FileParams.nQuality = 10
        nRet1 = ueye.is_ImageFile(hCam, ueye.IS_IMAGE_FILE_CMD_SAVE, FileParams, ueye.sizeof(FileParams))
        print('take photo')
        nRet1 = ueye.is_FreezeVideo(hCam, ueye.IS_DONT_WAIT)
    status_old = status_new
    start3 = datetime.datetime.now()

    time.sleep(0.3)
    print(Count)
    print(nRet)
    end3 = datetime.datetime.now()
    print("拍照執行時間：", end3 - start3)

    cut_rectangle()
    start2 = datetime.datetime.now()
    cnn()
    end2 = datetime.datetime.now()
    print("辨識執行時間：", end2 - start2)
    #end = datetime.datetime.now()

    #print("完整執行時間：", end - start)
    print('-----------------------------------------------------------')
    tf.reset_default_graph()

    # Press q if you want to end the loop
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

# ---------------------------------------------------------------------------------------------------------------------------------------
# Releases an image memory that was allocated using is_AllocImageMem() and removes it from the driver management
ueye.is_FreeImageMem(hCam, pcImageMemory, MemID)
# Disables the hCam camera handle and releases the data structures and memory areas taken up by the uEye camera
ueye.is_ExitCamera(hCam)
# Destroys the OpenCv windows
cv2.destroyAllWindows()