Dubhe/dubhe-visual-server/parser_service/python_io/lazy_load.py

# -*- coding: UTF-8 -*-
"""
 Copyright 2020 Tianshu AI Platform. All Rights Reserved.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 =============================================================
"""
from pathlib import Path
from typing import Union
import os

from utils.redis_utils import RedisInstance
from python_io.dictionary_watcher import start_run_watcher
from python_io.logfile_loader import Trace_Thread


class LazyLoad:
    def __init__(
            self,
            run: str,
            rundir: Union[str, Path],
    ):
        self.run = run
        self.rundir = rundir

    # 惰性加载，在初始化的时候加载目前日志中的所有数据
    def init_load(self, uid, cache_path):
        # 开启文件监听
        start_run_watcher(self.run, str(self.rundir), uid, cache_path)
        files = [f for f in self.rundir.glob("*") if f.is_file()]
        for file in files:
            # 设置每个文件的初始加载状态都为False
            # (redis不支持Boolen类型，存为0或1代替)
            RedisInstance.set("{}_{}_{}_is_finish".format(uid, self.run,
                              file.name), 0)
            current_size = os.path.getsize(str(file))
            Trace_Thread(self.run, file, current_size, uid, cache_path)\
                .start()