# -*- coding: UTF-8 -*- """ Copyright 2020 Tianshu AI Platform. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ============================================================= """ from pathlib import Path from typing import Union import os from utils.redis_utils import RedisInstance from python_io.dictionary_watcher import start_run_watcher from python_io.logfile_loader import Trace_Thread class LazyLoad: def __init__( self, run: str, rundir: Union[str, Path], ): self.run = run self.rundir = rundir # 惰性加载,在初始化的时候加载目前日志中的所有数据 def init_load(self, uid, cache_path): # 开启文件监听 start_run_watcher(self.run, str(self.rundir), uid, cache_path) files = [f for f in self.rundir.glob("*") if f.is_file()] for file in files: # 设置每个文件的初始加载状态都为False # (redis不支持Boolen类型,存为0或1代替) RedisInstance.set("{}_{}_{}_is_finish".format(uid, self.run, file.name), 0) current_size = os.path.getsize(str(file)) Trace_Thread(self.run, file, current_size, uid, cache_path)\ .start()