#!/usr/bin/env python # coding:utf-8 import io import json import os import sys import google.cloud.storage import requests curr_dir = os.path.split(os.path.abspath(__file__))[0] print(curr_dir) sys.path.append(os.path.join(curr_dir, "../")) import utils as utils import firebase_admin from firebase_admin import credentials, db from firebase_admin import storage, firestore from oauth2client.service_account import ServiceAccountCredentials DOF_GOOGLE_SERVER_FILE = os.path.join(curr_dir, "find-differences-65e47.json") D2_GOOGLE_SERVER_FILE = os.path.join(curr_dir, "dof2-b9070.json") FindOut_GOOGLE_SERVER_FILE = os.path.join(curr_dir, "dof2-b9070.json") FindMaster_GOOGLE_SERVER_FILE = os.path.join(curr_dir, "find-master-387702.json") FindIt_GOOGLE_SERVER_FILE = os.path.join(curr_dir, "find-it-a08e5.json") FindObject_GOOGLE_SERVER_FILE = os.path.join(curr_dir, "find-object-a08e5-f833603c7dd7.json") FoodSort_GOOGLE_SERVER_FILE = os.path.join(curr_dir, "food-sort-387702-712f402473f6.json") DOF_PROJECT_ID = "find-differences-65e47" D2_PROJECT_ID = "dof2-b9070" FindOut_PROJECT_ID = "dof2-b9070" FindMaster_PROJECT_ID = "find-master-88ffb" FindIt_PROJECT_ID = "find-it-a08e5" FindObject_PROJECT_ID = "find-object-45a0d" FoodSort_PROJECT_ID = "food-sort-1703a" BASE_URL = "https://firebaseremoteconfig.googleapis.com" def get_remote_config_file(project_id): return os.path.join(curr_dir, f"remote_config_{project_id}.json") def get_remote_config_url(project_id): return f"{BASE_URL}/v1/projects/{project_id}/remoteConfig" class Singleton: _instances = {} def __new__(cls, *args, **kwargs): if cls not in cls._instances: cls._instances[cls] = super().__new__(cls) return cls._instances[cls] class FirebaseHelperInstance(Singleton): def __init__(self): print("初始化") self.firebase_dof = None self.firebase_d2 = None self.firebase_find_out = None self.firebase_find_master = None self.firebase_find_it = None self.firebase_find_object = None self.firebase_food_sort = None def get_firebase_dof(self): if self.firebase_dof is None: self.firebase_dof = FirebaseHelper(DOF_PROJECT_ID, DOF_GOOGLE_SERVER_FILE) return self.firebase_dof def get_firebase_d2(self): if self.firebase_d2 is None: self.firebase_d2 = FirebaseHelper(D2_PROJECT_ID, D2_GOOGLE_SERVER_FILE) return self.firebase_d2 def get_firebase_find_out(self): if self.firebase_find_out is None: self.firebase_find_out = FirebaseHelper(FindOut_PROJECT_ID, FindOut_GOOGLE_SERVER_FILE) return self.firebase_find_out def get_firebase_find_master(self): if self.firebase_find_master is None: self.firebase_find_master = FirebaseHelper(FindMaster_PROJECT_ID, FindMaster_GOOGLE_SERVER_FILE) return self.firebase_find_master def get_firebase_find_it(self): if self.firebase_find_it is None: self.firebase_find_it = FirebaseHelper(FindIt_PROJECT_ID, FindIt_GOOGLE_SERVER_FILE) return self.firebase_find_it def get_firebase_find_object(self): if self.firebase_find_object is None: self.firebase_find_object = FirebaseHelper(FindObject_PROJECT_ID, FindObject_GOOGLE_SERVER_FILE) return self.firebase_find_object def get_firebase_food_sort(self): if self.firebase_food_sort is None: self.firebase_food_sort = FirebaseHelper(FindObject_PROJECT_ID, FindObject_GOOGLE_SERVER_FILE) return self.firebase_food_sort class FirebaseHelper: def __init__(self, project_id, google_service_file): print(f'init--{project_id}') self.firebase_app = None self.storage_instance = None self.config = {} self.config["project_id"] = project_id self.GOOGLE_SERVER_FILE = google_service_file self.REMOTE_CONFIG_FILE = get_remote_config_file(project_id) self.REMOTE_CONFIG_URL = get_remote_config_url(project_id) self.init() def init(self): self.init_firebase(self.config["project_id"]) self.init_storage(self.config["project_id"]) def init_firebase(self, project_id): storage_bucket = "gs://" + project_id + ".appspot.com" if project_id == FoodSort_PROJECT_ID: storage_bucket = "gs://" + project_id + ".firebasestorage.app" databaseURL = "https://" + project_id + ".firebaseio.com/" cred = credentials.Certificate(self.GOOGLE_SERVER_FILE) self.firebase_app = firebase_admin.initialize_app(cred, { "databaseURL": databaseURL, "storageBucket": storage_bucket }, name=project_id) self.refFirestore = firestore.client(app=self.firebase_app) print("初始 firebase 成功") def init_storage(self, project_id): bucket = '' if project_id == FoodSort_PROJECT_ID: bucket = project_id + ".firebasestorage.app" else: bucket = project_id + ".appspot.com" if self.firebase_app: self.storage_instance = storage.bucket(name=bucket, app=self.firebase_app) print("初始 storage 成功") def get_files(self, prefix=""): print("prefix = " + prefix) blobs = self.storage_instance.list_blobs(prefix=prefix) return blobs def get_files_match(self, prefix, match_glob): blobs = self.storage_instance.list_blobs(prefix=prefix, match_glob=match_glob) return blobs def get_files_all_versions(self, prefix=""): print("prefix = " + prefix) blobs = self.storage_instance.list_blobs(prefix=prefix, versions=True) return blobs def get_file(self, storage_file, generation=None): blob = self.storage_instance.get_blob(storage_file, generation=generation) return blob def get_file_generation(self, storage_file): blob = self.storage_instance.get_blob(storage_file) if blob is None: return None else: return blob.generation # 上传单个文件(会比较hash和md5码) def upload_single_bundle(self, local_file, storage_file, blob_dic, ext_meta={}, try_count=0): try_count = try_count + 1 if try_count > 3: return False try: blob = None if storage_file in blob_dic: blob = blob_dic[storage_file] is_same = False if blob is not None: local_hash = utils.calc_hash(local_file) local_md5 = utils.calc_md5(local_file) storage_hash = "" if blob.metadata is not None and 'hash' in blob.metadata: storage_hash = blob.metadata['hash'] storage_md5 = "" if blob.metadata is not None and 'md5' in blob.metadata: storage_md5 = blob.metadata['md5'] is_same = storage_hash == local_hash and storage_md5 == local_md5 if not is_same: print(f"{local_file} 上传中..") self.upload_file(local_file, storage_file, ext_meta) else: print(f"{local_file} hash和md5码和storage文件一致,不重复上传..") return True except Exception as e: print(local_file + " 上传失败,尝试重试,错误信息:" + repr(e)) self.upload_single_bundle(local_file, storage_file, blob_dic, ext_meta, try_count) def upload_thumbnail_image(self, local_file, storage_file, ext_meta={}, try_count=0): try_count = try_count + 1 if try_count > 3: return False try: self.upload_file(local_file, storage_file, ext_meta) return True except Exception as e: print(local_file + " 上传失败,尝试重试,错误信息:" + repr(e)) self.upload_thumbnail_image(local_file, storage_file, ext_meta, try_count) def upload_file(self, local_file, storage_file, ext_meta={}): try: upload_blob = self.storage_instance.blob(storage_file) meta = { 'hash': utils.calc_hash(local_file), 'md5': utils.calc_md5(local_file), } meta.update(ext_meta) upload_blob.metadata = meta if local_file.endswith(".json"): utils.gzip_file(local_file) upload_blob.content_encoding = "gzip" upload_blob.upload_from_filename(local_file + '.gz') os.unlink(local_file + '.gz') else: upload_blob.upload_from_filename(local_file) blob = self.storage_instance.get_blob(storage_file) print(local_file + " 上传成功 generation = {}".format(blob.generation)) return blob.generation except Exception as e: raise Exception(e) def update_metadata(self, storage_file, ext_meta={}): try: blob = self.storage_instance.get_blob(storage_file) if blob is not None: meta = blob.metadata meta.update(ext_meta) blob.metadata = meta blob.patch() print(storage_file + " 更新metadata成功") else: print(storage_file + " 文件不存在") except Exception as e: raise Exception(e) # region RemoteConfig更新 def get_access_token(self): file_value = "" with open(self.GOOGLE_SERVER_FILE, "r") as f: file_value = json.load(f) credentials = ServiceAccountCredentials.from_json_keyfile_dict(file_value, [ "https://www.googleapis.com/auth/firebase.remoteconfig"]) access_token_info = credentials.get_access_token() return access_token_info.access_token def get_remote_value(self): """ 获取RemoteConfig配置,并写入到remote_config.json中 :return: ETag """ try: headers = { "Authorization": "Bearer " + self.get_access_token() } resp = requests.get(self.REMOTE_CONFIG_URL, headers=headers) if resp.status_code == 200: with io.open(self.REMOTE_CONFIG_FILE, "wb") as f: f.write(resp.text.encode("utf-8")) print("remote config 写入完成: remote_config.json") print("ETag from server: {}".format(resp.headers["ETag"])) return resp.headers["ETag"] else: print("remote_config.json上传失败") print(resp.text) return None except Exception as e: print("获取 RemoteConfig值失败 " + repr(e)) raise Exception("Fail") def upload_remote_config_value(self, etag): """ 上传 remote_config.json 文件至firebase后台 :param: etag """ try: with open(self.REMOTE_CONFIG_FILE, "r", encoding="utf-8") as f: content = f.read() print("开始上传 remote config:>" + content + "<") headers = { "Authorization": "Bearer " + self.get_access_token(), "Content-Type": "application/json; UTF-8", "If-Match": etag } resp = requests.put(self.REMOTE_CONFIG_URL, data=content.encode("utf-8"), headers=headers) if resp.status_code == 200: print("推送成功") print("ETag from server: {}".format(resp.headers["ETag"])) return True else: print("推送失败") print(resp.text) return False except Exception as e: print("更新 RemoteConfig值失败 " + repr(e)) return False def check_or_create_string_field(self, json_dict, keys, value): arr = keys.split("/") field = json_dict index = 0 for item in arr: if index == len(arr) - 1: print("item = " + item) field[item] = value print(field[item]) else: field = field[item] index = index + 1 def check_or_create_json_value(self, json_dict, keys, value): arr = keys.split("/") field = json_dict index = 0 for item in arr: if index == len(arr) - 1: print("item = " + item) field[item] = json.dumps(value) print(field[item]) else: field = field[item] index = index + 1 def check_or_create_json_field_value(self, json_dict, keys, sub_key, sub_value): arr = keys.split("/") field = json_dict index = 0 for item in arr: if index == len(arr) - 1: print("item = " + item) value = json.loads(field[item]) value[sub_key] = sub_value field[item] = json.dumps(value) print(field[item]) else: field = field[item] index = index + 1 def update_remote_config_json_value(self, group, condition, remote_key, value, is_upload=False): try: etag = None if is_upload or not os.path.exists(self.REMOTE_CONFIG_FILE): etag = self.get_remote_value() remote_content = "" with open(self.REMOTE_CONFIG_FILE, "r") as f: remote_content = f.read() if remote_content != None and remote_content != "": remote_content_json = json.loads(remote_content) keys = "" if group is not None and group != "": keys = f"parameterGroups/{group}/parameters/" else: keys = f"parameters/" if condition is not None and condition != "": keys = keys + f"{remote_key}/conditionalValues/{condition}/value" else: keys = keys + f"{remote_key}/defaultValue/value" self.check_or_create_json_value(remote_content_json, keys, value) print("\n\n") print(remote_content_json) print("\n\n") # 将online_json写入到remote_config.json文件 utils.write_json(self.REMOTE_CONFIG_FILE, remote_content_json) if is_upload: self.upload_remote_config_value(etag) except Exception as e: print(e) raise Exception(f"[remote_config group:{group}, condition:{condition}, remote_key:{remote_key}] 更新失败") def update_remote_config_json_field_value(self, group, condition, remote_key, json_field_key, value, is_upload=False): try: etag = None if is_upload or not os.path.exists(self.REMOTE_CONFIG_FILE): etag = self.get_remote_value() remote_content = "" with open(self.REMOTE_CONFIG_FILE, "r") as f: remote_content = f.read() if remote_content != None and remote_content != "": remote_content_json = json.loads(remote_content) keys = "" if group is not None and group != "": keys = f"parameterGroups/{group}/parameters/" else: keys = f"parameters/" if condition is not None and condition != "": keys = keys + f"{remote_key}/conditionalValues/{condition}/value" else: keys = keys + f"{remote_key}/defaultValue/value" self.check_or_create_json_field_value(remote_content_json, keys, json_field_key, value) # 将online_json写入到remote_config.json文件 utils.write_json(self.REMOTE_CONFIG_FILE, remote_content_json) if is_upload: self.upload_remote_config_value(etag) except Exception as e: print(e) raise Exception(f"[remote_config group:{group}, condition:{condition}, remote_key:{remote_key}, json_field_key:{json_field_key}] 更新失败") def update_remote_config_string_value(self, group, condition, remote_key, value, is_upload=False): try: etag = None if is_upload or not os.path.exists(self.REMOTE_CONFIG_FILE): etag = self.get_remote_value() remote_content = "" with open(self.REMOTE_CONFIG_FILE, "r") as f: remote_content = f.read() if remote_content != None and remote_content != "": remote_content_json = json.loads(remote_content) keys = "" if group is not None and group != "": keys = f"parameterGroups/{group}/parameters/" else: keys = f"parameters/" if condition is not None and condition != "": keys = keys + f"{remote_key}/conditionalValues/{condition}/value" else: keys = keys + f"{remote_key}/defaultValue/value" self.check_or_create_string_field(remote_content_json, keys, value) print("\n\n") print(remote_content_json) print("\n\n") # 将online_json写入到remote_config.json文件 utils.write_json(self.REMOTE_CONFIG_FILE, remote_content_json) if is_upload: self.upload_remote_config_value(etag) except Exception as e: print(e) raise Exception(f"[remote_config group:{group}, condition:{condition}, remote_key:{remote_key}] 更新失败") def GetInfor(self, targetFile: str) -> google.cloud.storage.bucket.Bucket: blob = self.storage_instance.get_blob(targetFile) return blob def GetGeneration(self, targetFile: str) -> str: return str(self.GetInfor(targetFile).generation)