BBend-Test-JWT/manage.py

#啟動和管理項目
from distutils.log import error
from encodings import utf_8
from multiprocessing import Value
from app import create_app
from flask import render_template, request, session
import paramiko
import git
import shutil
from time import sleep as sl
from threading import Lock
import threading
from concurrent.futures import ThreadPoolExecutor
import csv
import os
import stat
import time


# app, db, mqtt, socketio = create_app() # Rita 0428
app, db, mqtt = create_app()
thread = None
thread_lock = Lock()

# Rita 0428
# @socketio.on('connect_event', namespace='/test_conn')
# def connected_msg(msg):
#     print("connect success")
#     print(msg)
#     socketio.emit('stdout', 'success', namespace='/test_conn')
#     print(socketio)

D = {"item_name":0, "command":0}

# 0411 shutil-rmtree 刪除 .get 唯讀檔
# https://stackoverflow.com/questions/21261132/shutil-rmtree-to-remove-readonly-files
def remove_readonly(func, path, _):
    os.chmod(path, stat.S_IWRITE)
    func(path)
    
# # 建置伺服器的訪問路徑 -- 舊 保留 0413 測試
# @app.route('/add_server', methods=['POST', 'GET'])
# def add_server_views():
#     username = session['uname']
#     L = []
#     if request.method == 'GET':
#         # 0407 測試註解一下
#         # repo = git.Repo.init(path='.')
#         # new_repo = git.Repo.clone_from(url='http://60.250.156.230:3000/benson/TestSSH.git', to_path='../new')
#         # new_repo = git.Repo.clone_from(url='http://60.250.156.230:3000/rita/CoffeeProject.git', to_path='../new')
#         os.system('git clone -b sh --single-branch http://60.250.156.230:3000/Automatic_build_system/Automatically_install_files.git C:\\Users\\USER\\Rita\\operating\\sixthproject\\new')
#         # with open('/home/benson/Project/new/All_installv3.2.sh') as f:
#         with open('../new/coffeesystem.sh', encoding='utf-8') as f:
#             # content = f.read().split('-Dividing line-')
#             # D['item_name'] = content[0].split('\n')[1:]
#             # D['command'] = content[1].split('\n')
#             D['command'] = f.read().split('\n')
#         # print("D['command']: ", D['command'])

#         # # 0407 測試中暫時註解
#         # # with open('/home/benson/Project/new/Item.txt') as f:
#         # with open('../new/Item.txt') as f:
#         #     D['item_name'] = f.read().split('\n')
#         #     for item in D['item_name']:
#         #         L.append('.'.join(item.split('.')[1:]))         # 拿掉 '.'.join 的話會變成 [['python3', '8'], ['python3-pip'],
        
        
#         # os.remove('C:/Users/USER/Rita/operating/sixthproject/new/.git')
#         # os.system('rm -rf ../new/.git')
#         # os.chmod('../new/.git', stat.S_IWRITE)
#         # os.unlink('../new/.git')
#         # sl(6)
#         # shutil.rmtree('../new', ignore_errors=True, onerror=del_rw)
#         sl(3)
#         shutil.rmtree('../new', onerror=remove_readonly)
#         sl(3)
#         # os.rmdir('../new')
#         # sl(6)
#         # #刪除非空資料夾
#         # try:
#         #     shutil.rmtree('../new', ignore_errors=True, onerror=del_rw)
#         #     # os.remove('../new')
#         #     print("=== DELETE ===")
#         #     sl(1)
#         # except OSError as e:
#         #     print(f"OSError:{e.strerror}")      # Error:存取被拒。
#         # # print("L: ", L)                         # L:  ['python3.8', 'python3-pip', 'git', ... , 'ROS-ubuntu-18.04-melodic']
        


#         # 0407 測試註解一下
#         # for i in range(0, len(D['item_name'])):
#         #     item = D['item_name'][i].split('=')[-1].replace('"','')
#         #     L.append(item)
#         # L = L[:-1]

#         # # 0411 測試
#         # D['command'] = [
#         #                 # 'cd /tmp/; pwd; sudo wget https://files.phpmyadmin.net/phpMyAdmin/5.1.0/phpMyAdmin-5.1.0-all-languages.tar.gz;tar -zxvf phpMyAdmin-5.1.0-all-languages.tar.gz;sudo rm phpMyAdmin-5.1.0-all-languages.tar.gz; sudo mv phpMyAdmin-5.1.0-all-languages /var/www/html/phpmyadmin;',
#         #                 'cd /var/www/html/phpmyadmin; pwd; mkdir tmp;',
#         #                 'cd /var/www/html/phpmyadmin; pwd; mkdir tmp;',
#         #                 'cd; pwd;',
#         #                 'pwd']

#         L2= []
#         for i in range(0, len(D['command'])):
#             if D['command'][i] == "'" or D['command'][i] == '' or "#" in D['command'][i] or D['command'][i] == ' ':
#                 continue
#             L2.append(D['command'][i])
#         D['command'] = L2
#         # print("D['command'] = L2: ", D['command'])

#         # print("=============== D['item_name'] ===============")
#         # print(D['item_name'])
#         # print("=============== D['command'] ===============")
#         # print(D['command'])
#         # print("=============== CODE CHECK 3 ===============")

#         # content = D['item_name'] + D['command']
#         # for i in content:
#         #     print(i)
#         return render_template('add_server.html', params=locals())
#     else:
#         item_install_fail = 0
#         d = request.form.to_dict()
#         ip = d['ip']
#         loginname = d['loginname']
#         username = d['username']
#         password = d['password']
#         print(ip)
#         print(loginname)
#         print(username)
#         print(password)
#         try:
#             # 建立一個sshclient物件
#             ssh = paramiko.SSHClient()
#             # 允許將信任的主機自動加入到host_allow 列表，此方法必須放在connect方法的前面
#             ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
#             # 指定本地的RSA私鑰檔案,如果建立金鑰對時設定的有密碼，password為設定的密碼，如無不用指定password引數
#             # pkey = paramiko.RSAKey.from_private_key_file('/home/super/.ssh/id_rsa', password='12345')
#             # pkey = paramiko.RSAKey.from_private_key_file('/home/ptop/topicjie/scripts/keys/id_rsa')
#             # 建立連線
#             # ssh.connect(hostname=ip,
#             #             port=22,
#             #             username='gs1801',
#             #             pkey=pkey)
#             # with open('/home/benson/test11.sh') as f:
#             #     content = f.read().split('\n')[0:-1]
#             # print(content)
#             # L = []
#             # for i in range(0, len(D['ssh_content'])):
#             #     if "#" in D['ssh_content'][i] or D['ssh_content'][i] == '' or D['ssh_content'][i] == ' ':
#             #         continue
#             #     L.append(D['ssh_content'][i])
#             # for i in L:
#             #     print(i)

#             ssh.connect(hostname=ip,
#                         port=22,
#                         username=loginname,
#                         password=password)
#             # 假如帳號密碼正確沒有被捕捉到異常，傳送訊息給前端
#             socketio.emit('user-confirm', 'pass', namespace='/test_conn')
#             # os.system("scp /home/benson/test10.sh gs1801@" + ip + ":/home/gs1801/")
#             # 執行命令
#             # stdin, stdout, stderr = ssh.exec_command("ls -l /home/gs1801;touch test7.py")
#             # stdin, stdout, stderr = ssh.exec_command("touch test10.sh;echo '" + content + "' >> test10.sh;bash test10.sh")
#             # print('test1')
#             # print(stdout.read().decode())
#             # print('test2')
#             # print(stderr.read().decode())

#             # content = D['item_name'] + D['command']
#             with open('ItemFile.csv', 'w', newline='') as csvFile:
#                 csvWriter = csv.writer(csvFile)
#                 csvWriter.writerow(['Command', 'Install Content', 'Time', 'Status'])

#             # # Rita 0406 TEST
#             D['command'] = [
#                             # 'sudo pip3 install cryptography==3.3.2
#                         ]

#             for command in D['command']:
#                 if '使用者' in command :
#                     command = command.replace('使用者', username)
#                 print(command)
#                 stdin, stdout, stderr = ssh.exec_command(command)
#                 command_start_time = time.time()
#                 # sl(6)      # Rita 原本為 sl(6) 增加後觀察是否 clone 輸出會改變
#                 # Rita 0406 stdout(success)、stderr(error) 每次只會輸出其中一個, 以此做為判斷, 例如：
#                 # print('=========================================')
#                 # print('stdout: ', stdout.read().decode())
#                 # print('stderr: ', stderr.read().decode())
#                 # print('=========================================')
#                 # 輸出為：
#                 # =========================================
#                 # stdout:  正在讀取套件清單...
#                 # 正在重建相依關係...
#                 # 正在讀取狀態資料...
#                 # python3.8 is already the newest version (3.8.0-3ubuntu1~18.04.2).
#                 # 升級 0 個，新安裝 0 個，移除 0 個，有 208 個未被升級。
#                 # stderr:
#                 # =========================================
                
#                 error = stderr.read().decode()
#                 success = stdout.read().decode()
#                 # print('error: ', error, '\nlen(error): ', len(error))
#                 print('error: ', error, '\nlen(error): ')
#                 print('success: ', success, '\nlen(success): ')
#                 if len(success) > 200000:
#                     success = success[0:20000] + '... etc'

#                 execution_time = round(time.time() - command_start_time, 3)
#                 if error:
#                     print('error')
#                     print(error)
#                     if item_install_fail == 0:
#                         item_install_fail = 1
#                     with open('ItemFile.csv', 'a', newline='') as csvFile:
#                         csvWriter = csv.writer(csvFile)
#                         csvWriter.writerow([command.replace(';', ';\n'), 
#                                             'success: \n' + success + '\n' + 'error: \n' + error, 
#                                             # 'success: \n' + success, 
#                                             # 'error: \n' + error, 
#                                             execution_time, 
#                                             'error'])
#                     socketio.emit('stderr', error, namespace='/test_conn')
#                 else:
#                     print('success')
#                     print(success)
#                     with open('ItemFile.csv', 'a', newline='') as csvFile:
#                         csvWriter = csv.writer(csvFile)
#                         csvWriter.writerow([command.replace(';', ';\n'), 
#                                             'success: \n' + success + '\nerror: \n' + error, 
#                                             execution_time, 
#                                             'success'])
#                     socketio.emit('stdout', success, namespace='/test_conn')

#                 sl(1)
#                 print('-----------------------------------------------------------------------------')

#             # Rita 0406 本來就註解
#             # def sshJob():
#             #     for command in content:
#             #         stdin, stdout, stderr = ssh.exec_command(command)
#             #     #
#             #     #     # print(stdout.read().decode())
#             #     #     # print('----------------------------------------')
#             #     #     # print(stderr.read().decode())
#             #     #
#             #         print(stdin)
#             #         if stderr.read().decode():
#             #             L.append(0)
#             #             print('error')
#             #             socketio.emit('stderr', stderr.read().decode())
#             #         else:
#             #             L.append(1)
#             #             print('success')
#             #             socketio.emit('stdout', stdout.read().decode())
#             #         sl(5)
#             #     ssh.close()

#             # Rita 0406 本來就註解
#             # def sshJob():
#             #     sl(5)
#             #     i = 0
#             #     total = len(content)
#             #     while True:
#             #         stdin, stdout, stderr = ssh.exec_command(content[i])
#             #     #
#             #     #     # print(stdout.read().decode())
#             #     #     # print('----------------------------------------')
#             #     #     # print(stderr.read().decode())
#             #     #
#             #         if stderr.read().decode():
#             #             L.append(0)
#             #             print('error')
#             #             socketio.emit('stderr', stderr.read().decode())
#             #         else:
#             #             L.append(1)
#             #             print('success')
#             #             socketio.emit('stdout', stdout.read().decode())
#             #         sl(5)
#             #         i += 1
#             #         if i == total:
#             #             break
#             #     ssh.close()

#             # ssh_thread = threading.Thread(target=sshJob)
#             # ssh_thread.daemon = True
#             # ssh_thread.start()

#             # global thread
#             # with thread_lock:
#             #     if thread is None:
#             #         thread = socketio.start_background_task(target=sshJob)

#             # print(L)
#             #
#             # item_list = ItemList()
#             # item_list.ip = ip
#             # item_list.pymysql = L[0]
#             # item_list.eventlet = L[1]
#             # datetime = dt.now()
#             # item_list.datetime = datetime
#             # db.session.add(item_list)
#             # db.session.commit()

#             # 關閉寫入的 CSV 檔案
#             csvFile.close()

#             # 關閉連線
#             ssh.close()
#             if item_install_fail:
#                 res = '有套件未安裝完成!'
#             else:
#                 res = '建置完成!'
#             return res
#         except Exception as e:
#             print(e)
#             return str(e)

Automatic_Build_System = {
    'python3.8': '0',
    'python3-pip': '0',
    'cryptography': '0',
    'upgrade': '0',
    'grep': '0',
    'git': '0',
    'gitpython': '0',
    'python3-tk': '0',
    'wget': '0',
    'python3-dev': '0',
    'default-libmysqlclient-dev': '0',
    'build-essential': '0',
    'tasksel': '0',
    'lamp-server': '0',
    'phpMyAdmin-5.1.0-all-languages.tar.gz': '0',
    'openssl': '0',
    'php-common': '0',
    'php-curl': '0',
    'php-json': '0',
    'php-mbstring': '0',
    'php-mysql': '0',
    'php-xml': '0',
    'php-zip': '0',
    'php7.2-mbstring': '0',
    'php7.2-xml': '0',
    'php7.2-mysql': '0',
    'php7.2-common': '0',
    'php7.2-curl': '0',
    'php7.2-json': '0',
    'php7.2-zip': '0',
    'apache2': '0',
    'paho-mqtt': '0',
    'mosquitto': '0',
    'mqtt_server_setting': '0',
    'rclocal/rc.local': '0',
    'golang': '0',
    'gogsfile': '0',
    'requirements': '0',
    'CoffeeProject': '0',
    'BBend-Test-JWT': '0',
    'Coffee_MySQL': '0',
    'chrony': '0',
    'ntpdate': '0',
    '--keyserver': '0',
    'update': '0',
    'ros-melodic-desktop-full': '0',
    'ros-melodic-rqt*': '0',
    'python-rosdep': '0',
    'python-rosinstall': '0',
    'python-rosinstall-generator': '0',
    'python-wstool': '0',
    'catkin_make': '0'
}

@app.route('/ABS_update', methods=['POST', 'GET'])
def ABS_update():
    return Automatic_Build_System

# 0413 測試中, 原版在上方
@app.route('/add_server', methods=['POST', 'GET'])
def add_server_views():
    username = session['uname']
    L = []
    CURRENT_PATH = os.path.dirname(__file__)
    print("CURRENT_PATH: ", CURRENT_PATH)
    
    if request.method == 'GET':
        with open(CURRENT_PATH + '/Item.txt') as f:
            D['item_name'] = f.read().split('\n')
            for item in D['item_name']:
                # L.append('.'.join(item.split('.')[1:]))         # 拿掉 '.'.join 的話會變成 [['python3', '8'], ['python3-pip'],
                L.append(item)

        # STATUS = session['status']
        return render_template('add_server.html', params=locals())
    else:
        item_install_fail = 0
        d = request.form.to_dict()
        ip = d['ip']
        loginname = d['loginname']
        username = d['username']
        password = d['password']
        item1 = d['item1_checked']
        item2 = d['item2_checked']
        item3 = d['item3_checked']
        item4 = d['item4_checked']
        item5 = d['item5_checked']
        item6 = d['item6_checked']
        item7 = d['item7_checked']
        item8 = d['item8_checked']
        print(d)
        try:
            # Rita 0428 測試
            # 建立一個sshclient物件
            ssh = paramiko.SSHClient()
            # 允許將信任的主機自動加入到host_allow 列表，此方法必須放在connect方法的前面
            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
            # 指定本地的RSA私鑰檔案,如果建立金鑰對時設定的有密碼，password為設定的密碼，如無不用指定password引數
            # pkey = paramiko.RSAKey.from_private_key_file('/home/super/.ssh/id_rsa', password='12345')
            # pkey = paramiko.RSAKey.from_private_key_file('/home/ptop/topicjie/scripts/keys/id_rsa')
            # 建立連線
            ssh.connect(hostname=ip,
                        port=22,
                        username=loginname,
                        password=password)
            # 假如帳號密碼正確沒有被捕捉到異常，傳送訊息給前端
            # socketio.emit('user-confirm', 'pass', namespace='/test_conn')

            with open(CURRENT_PATH + '/ItemFile.csv', 'w', newline='') as csvFile:
                csvWriter = csv.writer(csvFile)
                csvWriter.writerow(['Command', 'Install Content', 'Time', 'Status'])
            
            for item_num in range(1, 9):
                # if item1 == 'true':
                if eval("item" + str(item_num)) == 'true':
                    print('========== item ' + str(item_num) + ' install ====================================')
                    # 下載資料夾
                    os.system('git clone -b item' + str(item_num) + ' --single-branch http://60.250.156.230:3000/Automatic_build_system/Automatically_install_files.git ' + CURRENT_PATH + '/item' + str(item_num) + '')
                    with open( CURRENT_PATH + '/item' + str(item_num) + '/item' + str(item_num) + '.sh', encoding='utf-8') as f:
                        D['command'] = f.read().split('\n')
                    sl(1)
                    # 刪除下載資料夾
                    shutil.rmtree( CURRENT_PATH + '/item' + str(item_num) + '', onerror=remove_readonly)
                    
                    # 移除不需要執行之指令
                    sl(1)
                    L2= []
                    for i in range(0, len(D['command'])):
                        if D['command'][i] == "'" or D['command'][i] == '' or "#" in D['command'][i] or D['command'][i] == ' ':
                            continue
                        L2.append(D['command'][i])
                    D['command'] = L2


                    # 0510 Rita 測試
                    for command in D['command']:
                        if '使用者' in command :
                            command = command.replace('使用者', username)
                        if 'sudo apt-get install' in command :
                            command = command.replace('sudo apt-get install', "sudo DEBIAN_FRONTEND=noninteractive apt-get install")
                        print(command)
                        # 判斷若此 command 符合 Automatic_Build_System 的 Key, 則改變狀態
                        for key, value in Automatic_Build_System.items():
                            if key in command and value == '0':
                                # print('key: ', key, ' / command: ', command)
                                Automatic_Build_System[key] = "Install"

                        sl(1)
                        command_start_time = time.time()

                        # 0504 Rita 測試
                        stdin, stdout, stderr = ssh.exec_command(command)
                        error = stderr.read().decode()
                        success = stdout.read().decode()
                        # print('error: ', error, '\nlen(error): ', len(error))
                        print('error: ', error)
                        print('success: ', success)
                        if len(success) > 200000:
                            success = success[0:20000] + '... etc'
                            
                        execution_time = round(time.time() - command_start_time, 3)

                        if error:
                            # 排除 錯誤訊息中的 警告訊息
                            Warning_list = [
                                "WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager.", 
                                "Cloning into", 
                                "Warning: apt-key output should not be parsed (stdout is not a terminal)", 
                                "gpg", 
                                "Warning: running 'rosdep update' as root is not recommended.", 
                                "You should run",
                                "[Warning] Using a password on the command line interface can be insecure."
                            ]
                            error_command_list = error.split('\n')
                            for_error_command_list = error_command_list.copy()
                            for cl in range(0, len(error_command_list)):
                                for W in range(len(Warning_list)):
                                    try:
                                        if Warning_list[W] in error_command_list[cl]:
                                            for_error_command_list.remove(error_command_list[cl])
                                    except IndexError:
                                        pass
                            if for_error_command_list == [''] or for_error_command_list == []:
                                print('=== Warning ===')
                                with open(CURRENT_PATH + '/ItemFile.csv', 'a', newline='') as csvFile:
                                    csvWriter = csv.writer(csvFile)
                                    csvWriter.writerow([command.replace(';', ';\n'), 
                                                        'success: \n' + success + '\n' + 'error: \n' + error, 
                                                        execution_time, 
                                                        'warning'])
                                # 判斷若此 command 符合 Automatic_Build_System 的 Key, 則改變狀態
                                for key, value in Automatic_Build_System.items():
                                    if key in command:
                                        Automatic_Build_System[key] = execution_time
                            else:
                                print('for_error_command_list: ', for_error_command_list)
                                print('=== error ===')

                                if item_install_fail == 0:
                                    item_install_fail = 1
                                with open(CURRENT_PATH + '/ItemFile.csv', 'a', newline='') as csvFile:
                                    csvWriter = csv.writer(csvFile)
                                    csvWriter.writerow([command.replace(';', ';\n'), 
                                                        'success: \n' + success + '\n' + 'error: \n' + error, 
                                                        execution_time, 
                                                        'error'])
                                # 判斷若此 command 符合 Automatic_Build_System 的 Key, 則改變狀態
                                for key, value in Automatic_Build_System.items():
                                    if key in command:
                                        Automatic_Build_System[key] = "Error"
                        else:
                            print('=== success ===')

                            with open(CURRENT_PATH + '/ItemFile.csv', 'a', newline='') as csvFile:
                                csvWriter = csv.writer(csvFile)
                                csvWriter.writerow([command.replace(';', ';\n'), 
                                                    'success: \n' + success + '\nerror: \n' + error, 
                                                    execution_time, 
                                                    'success'])
                            # 判斷若此 command 符合 Automatic_Build_System 的 Key, 則改變狀態
                            for key, value in Automatic_Build_System.items():
                                if key in command:
                                    Automatic_Build_System[key] = execution_time
                        
                        sl(5)     # Rita 原本為 sl(6) 增加後觀察是否 clone 輸出會改變
                        print('-----------------------------------------------------------------------------')
            
            # stdin, stdout, stderr = ssh.exec_command("sudo systemctl restart rc-local.service")
            sl(5)
            # stdin, stdout, stderr = ssh.exec_command("sudo reboot")

            with open( CURRENT_PATH + '/ItemFile.csv', 'a', newline='') as csvFile:
                csvWriter = csv.writer(csvFile)
                csvWriter.writerow(['',
                                    'Automatic_Build_System: \n' + str(Automatic_Build_System),  
                                    '', 
                                    ''])

            # 關閉連線
            ssh.close()
            if item_install_fail:
                res = '有套件未安裝完成!'
            else:
                res = '建置完成!'
            return res
        except Exception as e:
            print(e)
            return str(e)



#mqtt發布
@app.route('/mqtt', methods=['GET','POST'])
def mqtt_views():
    import json
    dict = request.args.to_dict()
    json = json.dumps(dict)
    print(json)
    mqtt.publish('AISKY/AppleFarm/MK-G/b8:27:eb:f8:24:92', json)
    return "Publish done"


#mqtt訂閱
@mqtt.on_connect()
def handle_connect(client, userdata, flags, rc):
    mqtt.subscribe('AISKY/AppleFarm/MK-G/b8:27:eb:f8:24:92/Log')

#處理mqtt訂閱的信息
@mqtt.on_message()
def handle_mqtt_message(client, userdata, message):
    payload = message.payload.decode()
    # p = json.loads(payload)
    # print("-------msg-------")
    # print('name  :', p['name'])
    # print('email :', p['email'])
    print(payload)

#調用日誌訊息
@mqtt.on_log()
def handle_logging(client, userdata, level, buf):
    print(level, buf)

#自動關閉所有未使用、掛著的連接
@app.teardown_appcontext
def shutdown_session(exception=None):
        db.session.remove()

if __name__ == '__main__':
    # Rita 0406 socketio.run(app, debug=True, host='0.0.0.0', port=5014)
    # Rita 0406 app.run(debug=True, host='0.0.0.0', port=5014)
    # print("===== BBend =====")
    # socketio.run(app, debug=True, host='0.0.0.0', port=5014)
    app.run(debug=True, host='0.0.0.0', port=5014)