tests/fake_client_test.py

# -*- coding: utf-8 -*-

from random import random

from petrel_client.client import Client
from petrel_client.fake_client import FakeClient


def customized_get(self, *args, **kwargs):
    # type 有 s3、dfs、mc
    if self.type == 's3':
        # s3 的参数有 cluster, bucket, key
        cluster, bucket, key = args
    else:
        # 剩余类型的参数只包含 key
        cluster, bucket, key = 'unknow', 'unknow', args[0]

    if self.type == 'mc' and random() < 0.5:
        # 模拟缓存失效
        return None
    else:
        return b'x' * 1024


def customized_put(self, *args, **kwargs):
    if self.type == 's3':
        # s3 的参数有 cluster, bucket, key, body
        cluster, bucket, key, body = args
    else:
        # 剩余类型的参数只包含 key, body
        cluster, bucket, (key, body) = 'unknow', 'unknow', args

    return len(body)


FakeClient.customized_get = customized_get
FakeClient.customized_put = customized_put

client = Client("~/fake_client.conf")

urls = [
    'cluster1:s3://my-bucket/object.1',  # 从 cluster1 中读取
    'cluster2:s3://my-bucket/object.2',  # 从 cluster2 中读取
    's3://my-bucket/object.3',           # 若不指定 cluster，则从配置文件中指定的 default_cluster 中读取
    'file://tmp/xxx',                    # 从 DFS 中读取
    '/tmp/xxx',                          # 若不包含 's3:' 或 'file:'，从 DFS 中读取
]

for _ in range(1000):
    for url in urls:
        client.get(url)