Skip to content

Commit 2b1a5b6

Browse files
author
lijiachen
committed
capacity check
1 parent b1b7be5 commit 2b1a5b6

File tree

13 files changed

+319
-12
lines changed

13 files changed

+319
-12
lines changed

ucm/store/infra/status/status.h

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ class Status {
4242
ESERIALIZE = UC_MAKE_STATUS_CODE(6),
4343
EDESERIALIZE = UC_MAKE_STATUS_CODE(7),
4444
EUNSUPPORTED = UC_MAKE_STATUS_CODE(8),
45+
ENOSPACE = UC_MAKE_STATUS_CODE(9),
4546
#undef UC_MAKE_STATUS_CODE
4647
};
4748

@@ -101,7 +102,11 @@ class Status {
101102
static Status s{Code::EUNSUPPORTED};
102103
return s;
103104
}
104-
105+
static Status& NoSpace()
106+
{
107+
static Status s{Code::ENOSPACE};
108+
return s;
109+
}
105110
public:
106111
Status(const Status& status) { this->code_ = status.code_; }
107112
Status& operator=(const Status& status)

ucm/store/nfsstore/cc/api/nfsstore.cc

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ class NFSStoreImpl : public NFSStore {
3535
int32_t Setup(const Config& config)
3636
{
3737
auto status = this->spaceMgr_.Setup(config.storageBackends, config.kvcacheBlockSize,
38-
config.tempDumpDirEnable);
38+
config.tempDumpDirEnable, config.storageCapacity);
3939
if (status.Failure()) {
4040
UC_ERROR("Failed({}) to setup SpaceManager.", status);
4141
return status.Underlying();
@@ -120,6 +120,7 @@ class NFSStoreImpl : public NFSStore {
120120
UC_INFO("Set UC::TempDumpDirEnable to {}.", config.tempDumpDirEnable);
121121
UC_INFO("Set UC::HotnessInterval to {}.", config.hotnessInterval);
122122
UC_INFO("Set UC::HotnessEnable to {}.", config.hotnessEnable);
123+
UC_INFO("Set UC::storageCapacity to {}.", config.storageCapacity);
123124
}
124125

125126
private:

ucm/store/nfsstore/cc/api/nfsstore.h

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,13 +43,15 @@ class NFSStore : public CCStore {
4343
bool tempDumpDirEnable;
4444
bool hotnessEnable;
4545
size_t hotnessInterval;
46+
size_t storageCapacity;
4647

4748
Config(const std::vector<std::string>& storageBackends, const size_t kvcacheBlockSize,
4849
const bool transferEnable)
4950
: storageBackends{storageBackends}, kvcacheBlockSize{kvcacheBlockSize},
5051
transferEnable{transferEnable}, transferDeviceId{-1}, transferStreamNumber{32},
5152
transferIoSize{262144}, transferBufferNumber{512}, transferTimeoutMs{30000},
52-
tempDumpDirEnable{false}, hotnessEnable{true}, hotnessInterval{60}
53+
tempDumpDirEnable{false}, hotnessEnable{true}, hotnessInterval{60},
54+
storageCapacity{0}
5355
{
5456
}
5557
};

ucm/store/nfsstore/cc/domain/space/space_layout.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ class SpaceLayout {
3636
virtual Status Setup(const std::vector<std::string>& storageBackends) = 0;
3737
virtual std::string DataFileParent(const std::string& blockId, bool activated) const = 0;
3838
virtual std::string DataFilePath(const std::string& blockId, bool activated) const = 0;
39+
virtual std::string ClusterPropertyFilePath() const = 0;
3940
};
4041

4142
} // namespace UC

ucm/store/nfsstore/cc/domain/space/space_manager.cc

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ std::unique_ptr<SpaceLayout> MakeSpaceLayout(const bool tempDumpDirEnable)
4040
}
4141

4242
Status SpaceManager::Setup(const std::vector<std::string>& storageBackends, const size_t blockSize,
43-
const bool tempDumpDirEnable)
43+
const bool tempDumpDirEnable, const size_t storageCapacity)
4444
{
4545
if (blockSize == 0) {
4646
UC_ERROR("Invalid block size({}).", blockSize);
@@ -50,20 +50,25 @@ Status SpaceManager::Setup(const std::vector<std::string>& storageBackends, cons
5050
if (!this->layout_) { return Status::OutOfMemory(); }
5151
auto status = this->layout_->Setup(storageBackends);
5252
if (status.Failure()) { return status; }
53+
status = this->property_.Setup(this->layout_->ClusterPropertyFilePath());
54+
if (status.Failure()) { return status; }
5355
this->blockSize_ = blockSize;
56+
this->capacity_ = storageCapacity;
5457
return Status::OK();
5558
}
5659

57-
Status SpaceManager::NewBlock(const std::string& blockId) const
60+
Status SpaceManager::NewBlock(const std::string& blockId)
5861
{
62+
Status status = this->CapacityCheck();
63+
if (status.Failure()) { return status; }
5964
constexpr auto activated = true;
6065
auto parent = File::Make(this->layout_->DataFileParent(blockId, activated));
6166
auto file = File::Make(this->layout_->DataFilePath(blockId, activated));
6267
if (!parent || !file) {
6368
UC_ERROR("Failed to new block({}).", blockId);
6469
return Status::OutOfMemory();
6570
}
66-
auto status = parent->MkDir();
71+
status = parent->MkDir();
6772
if (status == Status::DuplicateKey()) { status = Status::OK(); }
6873
if (status.Failure()) {
6974
UC_ERROR("Failed({}) to new block({}).", status, blockId);
@@ -88,10 +93,11 @@ Status SpaceManager::NewBlock(const std::string& blockId) const
8893
UC_ERROR("Failed({}) to new block({}).", status, blockId);
8994
return status;
9095
}
96+
this->property_.IncreaseCapacity(this->blockSize_);
9197
return Status::OK();
9298
}
9399

94-
Status SpaceManager::CommitBlock(const std::string& blockId, bool success) const
100+
Status SpaceManager::CommitBlock(const std::string& blockId, bool success)
95101
{
96102
const auto activatedParent = this->layout_->DataFileParent(blockId, true);
97103
const auto activatedFile = this->layout_->DataFilePath(blockId, true);
@@ -112,6 +118,7 @@ Status SpaceManager::CommitBlock(const std::string& blockId, bool success) const
112118
if (status.Failure()) {
113119
UC_ERROR("Failed({}) to {} block({}).", status, success ? "commit" : "cancel", blockId);
114120
}
121+
this->property_.DecreaseCapacity(this->blockSize_);
115122
return status;
116123
}
117124

@@ -136,4 +143,17 @@ bool SpaceManager::LookupBlock(const std::string& blockId) const
136143

137144
const SpaceLayout* SpaceManager::GetSpaceLayout() const { return this->layout_.get(); }
138145

146+
Status SpaceManager::CapacityCheck() const
147+
{
148+
if (this->capacity_ == 0) { return Status::OK(); }
149+
150+
const size_t used = this->property_.GetCapacity();
151+
if (used > this->capacity_ - this->blockSize_) {
152+
UC_ERROR("Capacity is not enough, capacity: {}, current: {}, block size: {}.",
153+
this->capacity_, used, this->blockSize_);
154+
return Status::NoSpace();
155+
}
156+
return Status::OK();
157+
}
158+
139159
} // namespace UC

ucm/store/nfsstore/cc/domain/space/space_manager.h

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,22 +26,27 @@
2626

2727
#include <memory>
2828
#include "space_layout.h"
29+
#include "space_property.h"
2930
#include "status/status.h"
3031

3132
namespace UC {
3233

3334
class SpaceManager {
3435
public:
3536
Status Setup(const std::vector<std::string>& storageBackends, const size_t blockSize,
36-
const bool tempDumpDirEnable);
37-
Status NewBlock(const std::string& blockId) const;
38-
Status CommitBlock(const std::string& blockId, bool success = true) const;
37+
const bool tempDumpDirEnable, const size_t storageCapacity = 0);
38+
Status NewBlock(const std::string& blockId);
39+
Status CommitBlock(const std::string& blockId, bool success = true);
3940
bool LookupBlock(const std::string& blockId) const;
4041
const SpaceLayout* GetSpaceLayout() const;
4142

43+
private:
44+
Status CapacityCheck() const;
4245
private:
4346
std::unique_ptr<SpaceLayout> layout_;
47+
SpaceProperty property_;
4448
size_t blockSize_;
49+
size_t capacity_;
4550
};
4651

4752
} // namespace UC
Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
/**
2+
* MIT License
3+
*
4+
* Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved.
5+
*
6+
* Permission is hereby granted, free of charge, to any person obtaining a copy
7+
* of this software and associated documentation files (the "Software"), to deal
8+
* in the Software without restriction, including without limitation the rights
9+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10+
* copies of the Software, and to permit persons to whom the Software is
11+
* furnished to do so, subject to the following conditions:
12+
*
13+
* The above copyright notice and this permission notice shall be included in all
14+
* copies or substantial portions of the Software.
15+
*
16+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22+
* SOFTWARE.
23+
* */
24+
#include "space_property.h"
25+
#include <atomic>
26+
#include <chrono>
27+
#include <thread>
28+
#include "logger/logger.h"
29+
#include "file/file.h"
30+
31+
namespace UC {
32+
33+
static constexpr uint32_t Magic = (('S' << 16) | ('p' << 8) | 1);
34+
static constexpr size_t PropertySize = 256;
35+
36+
struct Property {
37+
std::atomic<uint32_t> magic;
38+
uint32_t padding;
39+
std::atomic<size_t> capacity;
40+
};
41+
static_assert(sizeof(Property) <= PropertySize, "Property take too much space");
42+
static_assert(std::atomic<uint32_t>::is_always_lock_free, "magic must be lock-free");
43+
static_assert(std::atomic<size_t>::is_always_lock_free, "capacity must be lock-free");
44+
45+
inline auto PropertyPtr(void* addr) { return (Property*)addr; }
46+
47+
SpaceProperty::~SpaceProperty()
48+
{
49+
if (this->addr_) {
50+
File::MUnmap(this->addr_, PropertySize);
51+
}
52+
this->addr_ = nullptr;
53+
}
54+
55+
Status SpaceProperty::Setup(const std::string& PropertyFilePath)
56+
{
57+
auto file = File::Make(PropertyFilePath);
58+
if (!file) { return Status::OutOfMemory(); }
59+
auto flags = IFile::OpenFlag::CREATE | IFile::OpenFlag::EXCL | IFile::OpenFlag::READ_WRITE;
60+
auto status = file->Open(flags);
61+
if (status.Success()) { return this->InitShmProperty(file.get()); }
62+
if (status == Status::DuplicateKey()) { return this->LoadShmProperty(file.get()); }
63+
return status;
64+
}
65+
66+
void SpaceProperty::IncreaseCapacity(const size_t delta)
67+
{
68+
PropertyPtr(this->addr_)->capacity += delta;
69+
}
70+
71+
void SpaceProperty::DecreaseCapacity(const size_t delta)
72+
{
73+
auto property = PropertyPtr(this->addr_);
74+
auto capacity = property->capacity.load(std::memory_order_acquire);
75+
while (capacity > delta) {
76+
if (property->capacity.compare_exchange_weak(capacity, capacity - delta, std::memory_order_acq_rel)) {
77+
return;
78+
}
79+
capacity = property->capacity.load(std::memory_order_acquire);
80+
}
81+
}
82+
83+
size_t SpaceProperty::GetCapacity() const
84+
{
85+
return PropertyPtr(this->addr_)->capacity.load(std::memory_order_relaxed);
86+
}
87+
88+
Status SpaceProperty::InitShmProperty(IFile* shmPropertyFile)
89+
{
90+
auto status = shmPropertyFile->Truncate(PropertySize);
91+
if (status.Failure()) { return status; }
92+
status = shmPropertyFile->MMap(this->addr_, PropertySize, true, true, true);
93+
if (status.Failure()) { return status; }
94+
std::fill_n((uint8_t*)this->addr_, PropertySize, 0);
95+
auto property = PropertyPtr(this->addr_);
96+
property->padding = 0;
97+
property->capacity = 0;
98+
property->magic = Magic;
99+
return Status::OK();
100+
}
101+
102+
Status SpaceProperty::LoadShmProperty(IFile* shmPropertyFile)
103+
{
104+
auto status = shmPropertyFile->Open(IFile::OpenFlag::READ_WRITE);
105+
if (status.Failure()) { return status; }
106+
constexpr auto retryInterval = std::chrono::milliseconds(100);
107+
constexpr auto maxTryTime = 100;
108+
auto tryTime = 0;
109+
IFile::FileStat stat;
110+
do {
111+
if (tryTime > maxTryTime) {
112+
UC_ERROR("Shm file({}) not ready.", shmPropertyFile->Path());
113+
return Status::Retry();
114+
}
115+
std::this_thread::sleep_for(retryInterval);
116+
status = shmPropertyFile->Stat(stat);
117+
if (status.Failure()) { return status; }
118+
tryTime++;
119+
} while (static_cast<size_t>(stat.st_size) != PropertySize);
120+
status = shmPropertyFile->MMap(this->addr_, PropertySize, true, true, true);
121+
if (status.Failure()) { return status; }
122+
auto property = PropertyPtr(this->addr_);
123+
tryTime = 0;
124+
do {
125+
if (property->magic == Magic) { break; }
126+
if (tryTime > maxTryTime) {
127+
UC_ERROR("Shm file({}) not ready.", shmPropertyFile->Path());
128+
return Status::Retry();
129+
}
130+
std::this_thread::sleep_for(retryInterval);
131+
tryTime++;
132+
} while (true);
133+
return Status::OK();
134+
}
135+
136+
} // namespace UC
Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/**
2+
* MIT License
3+
*
4+
* Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved.
5+
*
6+
* Permission is hereby granted, free of charge, to any person obtaining a copy
7+
* of this software and associated documentation files (the "Software"), to deal
8+
* in the Software without restriction, including without limitation the rights
9+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10+
* copies of the Software, and to permit persons to whom the Software is
11+
* furnished to do so, subject to the following conditions:
12+
*
13+
* The above copyright notice and this permission notice shall be included in all
14+
* copies or substantial portions of the Software.
15+
*
16+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22+
* SOFTWARE.
23+
* */
24+
25+
#ifndef UNIFIEDCACHE_SPACE_PROPERTY_H
26+
#define UNIFIEDCACHE_SPACE_PROPERTY_H
27+
28+
#include "file/ifile.h"
29+
#include "status/status.h"
30+
31+
namespace UC {
32+
33+
class SpaceProperty {
34+
public:
35+
~SpaceProperty();
36+
Status Setup(const std::string& propertyFilePath);
37+
void IncreaseCapacity(const size_t delta);
38+
void DecreaseCapacity(const size_t delta);
39+
size_t GetCapacity() const;
40+
41+
private:
42+
Status InitShmProperty(IFile* shmPropertyFile);
43+
Status LoadShmProperty(IFile* shmPropertyFile);
44+
45+
private:
46+
void* addr_{nullptr};
47+
};
48+
49+
} // namespace UC
50+
51+
#endif

ucm/store/nfsstore/cc/domain/space/space_shard_layout.cc

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,10 +111,15 @@ std::string SpaceShardLayout::StorageBackend(const std::string& blockId) const
111111
return this->storageBackends_[hasher(blockId) % this->storageBackends_.size()];
112112
}
113113

114-
std::vector<std::string> SpaceShardLayout::RelativeRoots() const { return {this->DataFileRoot()}; }
114+
std::vector<std::string> SpaceShardLayout::RelativeRoots() const {
115+
return {
116+
this->DataFileRoot(),
117+
this->ClusterFileRoot(),
118+
};
119+
}
115120

116121
std::string SpaceShardLayout::DataFileRoot() const { return "data"; }
117-
122+
std::string SpaceShardLayout::ClusterFileRoot() const { return "cluster"; }
118123
void SpaceShardLayout::ShardBlockId(const std::string& blockId, uint64_t& front,
119124
uint64_t& back) const
120125
{
@@ -123,4 +128,10 @@ void SpaceShardLayout::ShardBlockId(const std::string& blockId, uint64_t& front,
123128
back = id->back();
124129
}
125130

131+
std::string SpaceShardLayout::StorageBackend() const { return this->storageBackends_.front(); }
132+
std::string SpaceShardLayout::ClusterPropertyFilePath() const
133+
{
134+
return fmt::format("{}{}/{}.bin", this->StorageBackend(), this->ClusterFileRoot(), "uc_property.bin");
135+
}
136+
126137
} // namespace UC

0 commit comments

Comments
 (0)