Dubhe/dubhe-visual-server/parser_service/python_io/lazy_load.py

# -*- coding: UTF-8 -*-
#  Copyright 2020 Zhejiang Lab. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#  =============================================================
from pathlib import Path
from typing import Union
import os

from utils.redis_utils import RedisInstance
from python_io.dictionary_watcher import start_run_watcher
from python_io.logfile_loader import Trace_Thread


class LazyLoad:
    def __init__(
            self,
            run: str,
            rundir: Union[str, Path],
    ):
        self.run = run
        self.rundir = rundir

    # 惰性加载，在初始化的时候加载目前日志中的所有数据
    def init_load(self, uid, cache_path):
        # 开启文件监听
        start_run_watcher(self.run, str(self.rundir), uid, cache_path)
        files = [f for f in self.rundir.glob("*") if f.is_file()]
        for file in files:
            # 设置每个文件的初始加载状态都为False
            # (redis不支持Boolen类型，存为0或1代替)
            RedisInstance.set("{}_{}_{}_is_finish".format(uid, self.run,
                              file.name), 0)
            current_size = os.path.getsize(str(file))
            Trace_Thread(self.run, file, current_size, uid, cache_path)\
                .start()