config-generator/firebase_tools/firebase_helper.py

#!/usr/bin/env python
# coding:utf-8
import io
import json
import os
import sys
import google.cloud.storage
import requests

curr_dir = os.path.split(os.path.abspath(__file__))[0]
print(curr_dir)
sys.path.append(os.path.join(curr_dir, "../"))

import utils as utils
import firebase_admin
from firebase_admin import credentials, db
from firebase_admin import storage, firestore
from oauth2client.service_account import ServiceAccountCredentials

DOF_GOOGLE_SERVER_FILE = os.path.join(curr_dir, "find-differences-65e47.json")
D2_GOOGLE_SERVER_FILE = os.path.join(curr_dir, "dof2-b9070.json")
FindOut_GOOGLE_SERVER_FILE = os.path.join(curr_dir, "dof2-b9070.json")
FindMaster_GOOGLE_SERVER_FILE = os.path.join(curr_dir, "find-master-387702.json")
FindIt_GOOGLE_SERVER_FILE = os.path.join(curr_dir, "find-it-a08e5.json")
FindObject_GOOGLE_SERVER_FILE = os.path.join(curr_dir, "find-object-a08e5-f833603c7dd7.json")
FoodSort_GOOGLE_SERVER_FILE = os.path.join(curr_dir, "food-sort-387702-712f402473f6.json")

DOF_PROJECT_ID = "find-differences-65e47"
D2_PROJECT_ID = "dof2-b9070"
FindOut_PROJECT_ID = "dof2-b9070"
FindMaster_PROJECT_ID = "find-master-88ffb"
FindIt_PROJECT_ID = "find-it-a08e5"
FindObject_PROJECT_ID = "find-object-45a0d"
FoodSort_PROJECT_ID = "food-sort-1703a"

BASE_URL = "https://firebaseremoteconfig.googleapis.com"

def get_remote_config_file(project_id):
    return os.path.join(curr_dir, f"remote_config_{project_id}.json")

def get_remote_config_url(project_id):
    return f"{BASE_URL}/v1/projects/{project_id}/remoteConfig"

class Singleton:
    _instances = {}

    def __new__(cls, *args, **kwargs):
        if cls not in cls._instances:
            cls._instances[cls] = super().__new__(cls)
        return cls._instances[cls]

class FirebaseHelperInstance(Singleton):
    def __init__(self):
        print("初始化")
        self.firebase_dof = None
        self.firebase_d2 = None
        self.firebase_find_out = None
        self.firebase_find_master = None
        self.firebase_find_it = None
        self.firebase_find_object = None
        self.firebase_food_sort = None

    def get_firebase_dof(self):
        if self.firebase_dof is None:
            self.firebase_dof = FirebaseHelper(DOF_PROJECT_ID, DOF_GOOGLE_SERVER_FILE)
        return self.firebase_dof

    def get_firebase_d2(self):
        if self.firebase_d2 is None:
            self.firebase_d2 = FirebaseHelper(D2_PROJECT_ID, D2_GOOGLE_SERVER_FILE)
        return self.firebase_d2

    def get_firebase_find_out(self):
        if self.firebase_find_out is None:
            self.firebase_find_out = FirebaseHelper(FindOut_PROJECT_ID, FindOut_GOOGLE_SERVER_FILE)
        return self.firebase_find_out

    def get_firebase_find_master(self):
        if self.firebase_find_master is None:
            self.firebase_find_master = FirebaseHelper(FindMaster_PROJECT_ID, FindMaster_GOOGLE_SERVER_FILE)
        return self.firebase_find_master

    def get_firebase_find_it(self):
        if self.firebase_find_it is None:
            self.firebase_find_it = FirebaseHelper(FindIt_PROJECT_ID, FindIt_GOOGLE_SERVER_FILE)
        return self.firebase_find_it
    def get_firebase_find_object(self):
        if self.firebase_find_object is None:
            self.firebase_find_object = FirebaseHelper(FindObject_PROJECT_ID, FindObject_GOOGLE_SERVER_FILE)
        return self.firebase_find_object
    def get_firebase_food_sort(self):
        if self.firebase_food_sort is None:
            self.firebase_food_sort = FirebaseHelper(FindObject_PROJECT_ID, FindObject_GOOGLE_SERVER_FILE)
        return self.firebase_food_sort

class FirebaseHelper:
    def __init__(self, project_id, google_service_file):
        print(f'init--{project_id}')
        self.firebase_app = None
        self.storage_instance = None
        self.config = {}
        self.config["project_id"] = project_id
        self.GOOGLE_SERVER_FILE = google_service_file
        self.REMOTE_CONFIG_FILE = get_remote_config_file(project_id)
        self.REMOTE_CONFIG_URL = get_remote_config_url(project_id)
        self.init()

    def init(self):
        self.init_firebase(self.config["project_id"])
        self.init_storage(self.config["project_id"])

    def init_firebase(self, project_id):
        storage_bucket = "gs://" + project_id + ".appspot.com"
        if project_id == FoodSort_PROJECT_ID:
            storage_bucket = "gs://" + project_id + ".firebasestorage.app"
        databaseURL = "https://" + project_id + ".firebaseio.com/"
        cred = credentials.Certificate(self.GOOGLE_SERVER_FILE)
        self.firebase_app = firebase_admin.initialize_app(cred, {
            "databaseURL": databaseURL,
            "storageBucket": storage_bucket
        }, name=project_id)

        self.refFirestore = firestore.client(app=self.firebase_app)
        print("初始 firebase 成功")

    def init_storage(self, project_id):
        bucket = ''
        if project_id == FoodSort_PROJECT_ID:
            bucket = project_id + ".firebasestorage.app"
        else:
            bucket = project_id + ".appspot.com"
        if self.firebase_app:
            self.storage_instance = storage.bucket(name=bucket, app=self.firebase_app)
            print("初始 storage 成功")

    def get_files(self, prefix=""):
        print("prefix = " + prefix)
        blobs = self.storage_instance.list_blobs(prefix=prefix)
        return blobs

    def get_files_match(self, prefix, match_glob):
        blobs = self.storage_instance.list_blobs(prefix=prefix, match_glob=match_glob)
        return blobs

    def get_files_all_versions(self, prefix=""):
        print("prefix = " + prefix)
        blobs = self.storage_instance.list_blobs(prefix=prefix, versions=True)
        return blobs

    def get_file(self, storage_file, generation=None):
        blob = self.storage_instance.get_blob(storage_file, generation=generation)
        return blob

    def get_file_generation(self, storage_file):
        blob = self.storage_instance.get_blob(storage_file)
        if blob is None:
            return None
        else:
            return blob.generation

    # 上传单个文件（会比较hash和md5码）
    def upload_single_bundle(self, local_file, storage_file, blob_dic, ext_meta={}, try_count=0):
        try_count = try_count + 1
        if try_count > 3:
            return False
        try:
            blob = None
            if storage_file in blob_dic:
                blob = blob_dic[storage_file]

            is_same = False
            if blob is not None:
                local_hash = utils.calc_hash(local_file)
                local_md5 = utils.calc_md5(local_file)

                storage_hash = ""
                if blob.metadata is not None and 'hash' in blob.metadata:
                    storage_hash = blob.metadata['hash']

                storage_md5 = ""
                if blob.metadata is not None and 'md5' in blob.metadata:
                    storage_md5 = blob.metadata['md5']

                is_same = storage_hash == local_hash and storage_md5 == local_md5

            if not is_same:
                print(f"{local_file} 上传中..")
                self.upload_file(local_file, storage_file, ext_meta)
            else:
                print(f"{local_file} hash和md5码和storage文件一致，不重复上传..")
            return True
        except Exception as e:
            print(local_file + " 上传失败，尝试重试，错误信息：" + repr(e))
            self.upload_single_bundle(local_file, storage_file, blob_dic, ext_meta, try_count)

    def upload_thumbnail_image(self, local_file, storage_file, ext_meta={}, try_count=0):
        try_count = try_count + 1
        if try_count > 3:
            return False
        try:
            self.upload_file(local_file, storage_file, ext_meta)
            return True
        except Exception as e:
            print(local_file + " 上传失败，尝试重试，错误信息：" + repr(e))
            self.upload_thumbnail_image(local_file, storage_file, ext_meta, try_count)

    def upload_file(self, local_file, storage_file, ext_meta={}):
        try:
            upload_blob = self.storage_instance.blob(storage_file)
            meta = {
                'hash': utils.calc_hash(local_file),
                'md5': utils.calc_md5(local_file),
            }

            meta.update(ext_meta)
            upload_blob.metadata = meta
            if local_file.endswith(".json"):
                utils.gzip_file(local_file)
                upload_blob.content_encoding = "gzip"
                upload_blob.upload_from_filename(local_file + '.gz')
                os.unlink(local_file + '.gz')
            else:
                upload_blob.upload_from_filename(local_file)
            blob = self.storage_instance.get_blob(storage_file)
            print(local_file + "   上传成功    generation = {}".format(blob.generation))
            return blob.generation
        except Exception as e:
            raise Exception(e)

    def update_metadata(self, storage_file, ext_meta={}):
        try:
            blob = self.storage_instance.get_blob(storage_file)
            if blob is not None:
                meta = blob.metadata
                meta.update(ext_meta)
                blob.metadata = meta
                blob.patch()
                print(storage_file + "   更新metadata成功")
            else:
                print(storage_file + "   文件不存在")
        except Exception as e:
            raise Exception(e)

    # region RemoteConfig更新

    def get_access_token(self):
        file_value = ""
        with open(self.GOOGLE_SERVER_FILE, "r") as f:
            file_value = json.load(f)

        credentials = ServiceAccountCredentials.from_json_keyfile_dict(file_value, [
            "https://www.googleapis.com/auth/firebase.remoteconfig"])
        access_token_info = credentials.get_access_token()
        return access_token_info.access_token

    def get_remote_value(self):
        """
        获取RemoteConfig配置，并写入到remote_config.json中
        :return: ETag
        """
        try:
            headers = {
                "Authorization": "Bearer " + self.get_access_token()
            }
            resp = requests.get(self.REMOTE_CONFIG_URL, headers=headers)

            if resp.status_code == 200:
                with io.open(self.REMOTE_CONFIG_FILE, "wb") as f:
                    f.write(resp.text.encode("utf-8"))

                print("remote config 写入完成: remote_config.json")
                print("ETag from server: {}".format(resp.headers["ETag"]))
                return resp.headers["ETag"]
            else:
                print("remote_config.json上传失败")
                print(resp.text)
                return None
        except Exception as e:
            print("获取 RemoteConfig值失败   " + repr(e))
            raise Exception("Fail")

    def upload_remote_config_value(self, etag):
        """
        上传 remote_config.json 文件至firebase后台
        :param: etag
        """
        try:
            with open(self.REMOTE_CONFIG_FILE, "r", encoding="utf-8") as f:
                content = f.read()
            print("开始上传 remote config:>" + content + "<")
            headers = {
                "Authorization": "Bearer " + self.get_access_token(),
                "Content-Type": "application/json; UTF-8",
                "If-Match": etag
            }
            resp = requests.put(self.REMOTE_CONFIG_URL, data=content.encode("utf-8"), headers=headers)
            if resp.status_code == 200:
                print("推送成功")
                print("ETag from server: {}".format(resp.headers["ETag"]))
                return True
            else:
                print("推送失败")
                print(resp.text)
                return False
        except Exception as e:
            print("更新 RemoteConfig值失败   " + repr(e))
            return False

    def check_or_create_string_field(self, json_dict, keys, value):
        arr = keys.split("/")
        field = json_dict
        index = 0
        for item in arr:
            if index == len(arr) - 1:
                print("item = " + item)
                field[item] = value
                print(field[item])
            else:
                field = field[item]

            index = index + 1

    def check_or_create_json_value(self, json_dict, keys, value):
        arr = keys.split("/")
        field = json_dict
        index = 0
        for item in arr:
            if index == len(arr) - 1:
                print("item = " + item)
                field[item] = json.dumps(value)
                print(field[item])
            else:
                field = field[item]

            index = index + 1

    def check_or_create_json_field_value(self, json_dict, keys, sub_key, sub_value):
        arr = keys.split("/")
        field = json_dict
        index = 0
        for item in arr:
            if index == len(arr) - 1:
                print("item = " + item)
                value = json.loads(field[item])
                value[sub_key] = sub_value
                field[item] = json.dumps(value)
                print(field[item])
            else:
                field = field[item]

            index = index + 1

    def update_remote_config_json_value(self, group, condition, remote_key, value, is_upload=False):
        try:
            etag = None
            if is_upload or not os.path.exists(self.REMOTE_CONFIG_FILE):
                etag = self.get_remote_value()

            remote_content = ""
            with open(self.REMOTE_CONFIG_FILE, "r") as f:
                remote_content = f.read()

            if remote_content != None and remote_content != "":
                remote_content_json = json.loads(remote_content)
                keys = ""
                if group is not None and group != "":
                    keys = f"parameterGroups/{group}/parameters/"
                else:
                    keys = f"parameters/"

                if condition is not None and condition != "":
                    keys = keys + f"{remote_key}/conditionalValues/{condition}/value"
                else:
                    keys = keys + f"{remote_key}/defaultValue/value"

                self.check_or_create_json_value(remote_content_json, keys, value)

                print("\n\n")
                print(remote_content_json)
                print("\n\n")
                # 将online_json写入到remote_config.json文件
                utils.write_json(self.REMOTE_CONFIG_FILE, remote_content_json)
                if is_upload:
                    self.upload_remote_config_value(etag)
        except Exception as e:
            print(e)
            raise Exception(f"[remote_config group:{group}, condition:{condition}, remote_key:{remote_key}] 更新失败")

    def update_remote_config_json_field_value(self, group, condition, remote_key, json_field_key, value, is_upload=False):
        try:
            etag = None
            if is_upload or not os.path.exists(self.REMOTE_CONFIG_FILE):
                etag = self.get_remote_value()

            remote_content = ""
            with open(self.REMOTE_CONFIG_FILE, "r") as f:
                remote_content = f.read()

            if remote_content != None and remote_content != "":
                remote_content_json = json.loads(remote_content)
                keys = ""
                if group is not None and group != "":
                    keys = f"parameterGroups/{group}/parameters/"
                else:
                    keys = f"parameters/"

                if condition is not None and condition != "":
                    keys = keys + f"{remote_key}/conditionalValues/{condition}/value"
                else:
                    keys = keys + f"{remote_key}/defaultValue/value"

                self.check_or_create_json_field_value(remote_content_json, keys, json_field_key, value)

                # 将online_json写入到remote_config.json文件
                utils.write_json(self.REMOTE_CONFIG_FILE, remote_content_json)
                if is_upload:
                    self.upload_remote_config_value(etag)
        except Exception as e:
            print(e)
            raise Exception(f"[remote_config group:{group}, condition:{condition}, remote_key:{remote_key}, json_field_key:{json_field_key}] 更新失败")

    def update_remote_config_string_value(self, group, condition, remote_key, value, is_upload=False):
        try:
            etag = None
            if is_upload or not os.path.exists(self.REMOTE_CONFIG_FILE):
                etag = self.get_remote_value()

            remote_content = ""
            with open(self.REMOTE_CONFIG_FILE, "r") as f:
                remote_content = f.read()

            if remote_content != None and remote_content != "":
                remote_content_json = json.loads(remote_content)
                keys = ""
                if group is not None and group != "":
                    keys = f"parameterGroups/{group}/parameters/"
                else:
                    keys = f"parameters/"

                if condition is not None and condition != "":
                    keys = keys + f"{remote_key}/conditionalValues/{condition}/value"
                else:
                    keys = keys + f"{remote_key}/defaultValue/value"

                self.check_or_create_string_field(remote_content_json, keys, value)

                print("\n\n")
                print(remote_content_json)
                print("\n\n")
                # 将online_json写入到remote_config.json文件
                utils.write_json(self.REMOTE_CONFIG_FILE, remote_content_json)
                if is_upload:
                    self.upload_remote_config_value(etag)
        except Exception as e:
            print(e)
            raise Exception(f"[remote_config group:{group}, condition:{condition}, remote_key:{remote_key}] 更新失败")

    def GetInfor(self, targetFile: str) -> google.cloud.storage.bucket.Bucket:

        blob = self.storage_instance.get_blob(targetFile)
        return blob

    def GetGeneration(self, targetFile: str) -> str:
        return str(self.GetInfor(targetFile).generation)