Skip to content

Commit c1b5ee1

Browse files
author
lijiachen
committed
capacity check
1 parent b53b23a commit c1b5ee1

File tree

13 files changed

+308
-10
lines changed

13 files changed

+308
-10
lines changed

ucm/store/infra/logger/logger.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@
2727
#include <cstddef>
2828
#include <fmt/format.h>
2929
#include <string>
30+
#include <chrono>
31+
#include <thread>
3032

3133
namespace UC::Logger {
3234

ucm/store/infra/status/status.h

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ class Status {
4242
ESERIALIZE = UC_MAKE_STATUS_CODE(6),
4343
EDESERIALIZE = UC_MAKE_STATUS_CODE(7),
4444
EUNSUPPORTED = UC_MAKE_STATUS_CODE(8),
45+
ENOSPACE = UC_MAKE_STATUS_CODE(9),
4546
#undef UC_MAKE_STATUS_CODE
4647
};
4748

@@ -101,7 +102,11 @@ class Status {
101102
static Status s{Code::EUNSUPPORTED};
102103
return s;
103104
}
104-
105+
static Status& NoSpace()
106+
{
107+
static Status s{Code::ENOSPACE};
108+
return s;
109+
}
105110
public:
106111
Status(const Status& status) { this->code_ = status.code_; }
107112
Status& operator=(const Status& status)

ucm/store/nfsstore/cc/api/nfsstore.cc

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ class NFSStoreImpl : public NFSStore {
3434
int32_t Setup(const Config& config)
3535
{
3636
auto status = this->spaceMgr_.Setup(config.storageBackends, config.kvcacheBlockSize,
37-
config.tempDumpDirEnable);
37+
config.tempDumpDirEnable, config.storageCapacity);
3838
if (status.Failure()) {
3939
UC_ERROR("Failed({}) to setup SpaceManager.", status);
4040
return status.Underlying();
@@ -105,6 +105,7 @@ class NFSStoreImpl : public NFSStore {
105105
UC_INFO("Set UC::BufferNumber to {}.", config.transferBufferNumber);
106106
UC_INFO("Set UC::TimeoutMs to {}.", config.transferTimeoutMs);
107107
UC_INFO("Set UC::TempDumpDirEnable to {}.", config.tempDumpDirEnable);
108+
UC_INFO("Set UC::storageCapacity to {}.", config.storageCapacity);
108109
}
109110

110111
private:

ucm/store/nfsstore/cc/api/nfsstore.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,13 +41,14 @@ class NFSStore : public CCStore {
4141
size_t transferBufferNumber;
4242
size_t transferTimeoutMs;
4343
bool tempDumpDirEnable;
44+
size_t storageCapacity;
4445

4546
Config(const std::vector<std::string>& storageBackends, const size_t kvcacheBlockSize,
4647
const bool transferEnable)
4748
: storageBackends{storageBackends}, kvcacheBlockSize{kvcacheBlockSize},
4849
transferEnable{transferEnable}, transferDeviceId{-1}, transferStreamNumber{32},
4950
transferIoSize{262144}, transferBufferNumber{512}, transferTimeoutMs{30000},
50-
tempDumpDirEnable{false}
51+
tempDumpDirEnable{false}, storageCapacity{0}
5152
{
5253
}
5354
};

ucm/store/nfsstore/cc/domain/space/space_layout.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626

2727
#include <string>
2828
#include <vector>
29+
#include <fmt/format.h>
2930
#include "status/status.h"
3031

3132
namespace UC {
@@ -36,6 +37,7 @@ class SpaceLayout {
3637
virtual Status Setup(const std::vector<std::string>& storageBackends) = 0;
3738
virtual std::string DataFileParent(const std::string& blockId, bool activated) const = 0;
3839
virtual std::string DataFilePath(const std::string& blockId, bool activated) const = 0;
40+
virtual std::string PropertyFilePath() const = 0;
3941
};
4042

4143
} // namespace UC

ucm/store/nfsstore/cc/domain/space/space_manager.cc

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ std::unique_ptr<SpaceLayout> MakeSpaceLayout(const bool tempDumpDirEnable)
4040
}
4141

4242
Status SpaceManager::Setup(const std::vector<std::string>& storageBackends, const size_t blockSize,
43-
const bool tempDumpDirEnable)
43+
const bool tempDumpDirEnable, const size_t storageCapacity)
4444
{
4545
if (blockSize == 0) {
4646
UC_ERROR("Invalid block size({}).", blockSize);
@@ -50,20 +50,25 @@ Status SpaceManager::Setup(const std::vector<std::string>& storageBackends, cons
5050
if (!this->layout_) { return Status::OutOfMemory(); }
5151
auto status = this->layout_->Setup(storageBackends);
5252
if (status.Failure()) { return status; }
53+
status = this->property_.Setup(this->layout_->PropertyFilePath());
54+
if (status.Failure()) { return status; }
5355
this->blockSize_ = blockSize;
56+
this->capacity_ = storageCapacity;
5457
return Status::OK();
5558
}
5659

57-
Status SpaceManager::NewBlock(const std::string& blockId) const
60+
Status SpaceManager::NewBlock(const std::string& blockId)
5861
{
62+
Status status = this->CapacityCheck();
63+
if (status.Failure()) { return status; }
5964
constexpr auto activated = true;
6065
auto parent = File::Make(this->layout_->DataFileParent(blockId, activated));
6166
auto file = File::Make(this->layout_->DataFilePath(blockId, activated));
6267
if (!parent || !file) {
6368
UC_ERROR("Failed to new block({}).", blockId);
6469
return Status::OutOfMemory();
6570
}
66-
auto status = parent->MkDir();
71+
status = parent->MkDir();
6772
if (status == Status::DuplicateKey()) { status = Status::OK(); }
6873
if (status.Failure()) {
6974
UC_ERROR("Failed({}) to new block({}).", status, blockId);
@@ -88,10 +93,11 @@ Status SpaceManager::NewBlock(const std::string& blockId) const
8893
UC_ERROR("Failed({}) to new block({}).", status, blockId);
8994
return status;
9095
}
96+
this->property_.IncreaseCapacity(this->blockSize_);
9197
return Status::OK();
9298
}
9399

94-
Status SpaceManager::CommitBlock(const std::string& blockId, bool success) const
100+
Status SpaceManager::CommitBlock(const std::string& blockId, bool success)
95101
{
96102
const auto activatedParent = this->layout_->DataFileParent(blockId, true);
97103
const auto activatedFile = this->layout_->DataFilePath(blockId, true);
@@ -112,6 +118,7 @@ Status SpaceManager::CommitBlock(const std::string& blockId, bool success) const
112118
if (status.Failure()) {
113119
UC_ERROR("Failed({}) to {} block({}).", status, success ? "commit" : "cancel", blockId);
114120
}
121+
this->property_.DecreaseCapacity(this->blockSize_);
115122
return status;
116123
}
117124

@@ -136,4 +143,17 @@ bool SpaceManager::LookupBlock(const std::string& blockId) const
136143

137144
const SpaceLayout* SpaceManager::GetSpaceLayout() const { return this->layout_.get(); }
138145

146+
Status SpaceManager::CapacityCheck() const
147+
{
148+
if (this->capacity_ == 0) { return Status::OK(); }
149+
150+
const size_t used = this->property_.GetCapacity();
151+
if (used > this->capacity_ - this->blockSize_) {
152+
UC_ERROR("Capacity is not enough, capacity: {}, current: {}, block size: {}.",
153+
this->capacity_, used, this->blockSize_);
154+
return Status::NoSpace();
155+
}
156+
return Status::OK();
157+
}
158+
139159
} // namespace UC

ucm/store/nfsstore/cc/domain/space/space_manager.h

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,22 +26,27 @@
2626

2727
#include <memory>
2828
#include "space_layout.h"
29+
#include "space_property.h"
2930
#include "status/status.h"
3031

3132
namespace UC {
3233

3334
class SpaceManager {
3435
public:
3536
Status Setup(const std::vector<std::string>& storageBackends, const size_t blockSize,
36-
const bool tempDumpDirEnable);
37-
Status NewBlock(const std::string& blockId) const;
38-
Status CommitBlock(const std::string& blockId, bool success = true) const;
37+
const bool tempDumpDirEnable, const size_t storageCapacity = 0);
38+
Status NewBlock(const std::string& blockId);
39+
Status CommitBlock(const std::string& blockId, bool success = true);
3940
bool LookupBlock(const std::string& blockId) const;
4041
const SpaceLayout* GetSpaceLayout() const;
4142

43+
private:
44+
Status CapacityCheck() const;
4245
private:
4346
std::unique_ptr<SpaceLayout> layout_;
47+
SpaceProperty property_;
4448
size_t blockSize_;
49+
size_t capacity_;
4550
};
4651

4752
} // namespace UC
Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
/**
2+
* MIT License
3+
*
4+
* Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved.
5+
*
6+
* Permission is hereby granted, free of charge, to any person obtaining a copy
7+
* of this software and associated documentation files (the "Software"), to deal
8+
* in the Software without restriction, including without limitation the rights
9+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10+
* copies of the Software, and to permit persons to whom the Software is
11+
* furnished to do so, subject to the following conditions:
12+
*
13+
* The above copyright notice and this permission notice shall be included in all
14+
* copies or substantial portions of the Software.
15+
*
16+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22+
* SOFTWARE.
23+
* */
24+
#include "space_property.h"
25+
#include <atomic>
26+
#include "logger/logger.h"
27+
#include "file/file.h"
28+
29+
namespace UC {
30+
31+
static constexpr uint32_t Magic = (('S' << 16) | ('p' << 8) | 1);
32+
static constexpr size_t PropertySize = 256;
33+
34+
struct Property {
35+
std::atomic<uint32_t> magic;
36+
uint32_t padding;
37+
std::atomic<size_t> capacity;
38+
};
39+
static_assert(sizeof(Property) <= PropertySize, "Property take too much space");
40+
static_assert(std::atomic<uint32_t>::is_always_lock_free, "magic must be lock-free");
41+
static_assert(std::atomic<size_t>::is_always_lock_free, "capacity must be lock-free");
42+
43+
inline auto PropertyPtr(void* addr) { return (Property*)addr; }
44+
45+
SpaceProperty::~SpaceProperty()
46+
{
47+
if (this->addr_) {
48+
File::MUnmap(this->addr_, PropertySize);
49+
}
50+
this->addr_ = nullptr;
51+
}
52+
53+
Status SpaceProperty::Setup(const std::string& propertyFilePath)
54+
{
55+
auto file = File::Make(propertyFilePath);
56+
if (!file) { return Status::OutOfMemory(); }
57+
auto flags = IFile::OpenFlag::CREATE | IFile::OpenFlag::EXCL | IFile::OpenFlag::READ_WRITE;
58+
auto status = file->Open(flags);
59+
if (status.Success()) { return this->InitShmProperty(file.get()); }
60+
if (status == Status::DuplicateKey()) { return this->LoadShmProperty(file.get()); }
61+
return status;
62+
}
63+
64+
void SpaceProperty::IncreaseCapacity(const size_t delta)
65+
{
66+
PropertyPtr(this->addr_)->capacity += delta;
67+
}
68+
69+
void SpaceProperty::DecreaseCapacity(const size_t delta)
70+
{
71+
auto property = PropertyPtr(this->addr_);
72+
auto capacity = property->capacity.load(std::memory_order_acquire);
73+
while (capacity > delta) {
74+
if (property->capacity.compare_exchange_weak(capacity, capacity - delta, std::memory_order_acq_rel)) {
75+
return;
76+
}
77+
capacity = property->capacity.load(std::memory_order_acquire);
78+
}
79+
}
80+
81+
size_t SpaceProperty::GetCapacity() const
82+
{
83+
return PropertyPtr(this->addr_)->capacity.load(std::memory_order_relaxed);
84+
}
85+
86+
Status SpaceProperty::InitShmProperty(IFile* shmPropertyFile)
87+
{
88+
auto status = shmPropertyFile->Truncate(PropertySize);
89+
if (status.Failure()) { return status; }
90+
status = shmPropertyFile->MMap(this->addr_, PropertySize, true, true, true);
91+
if (status.Failure()) { return status; }
92+
std::fill_n((uint8_t*)this->addr_, PropertySize, 0);
93+
auto property = PropertyPtr(this->addr_);
94+
property->padding = 0;
95+
property->capacity = 0;
96+
property->magic = Magic;
97+
return Status::OK();
98+
}
99+
100+
Status SpaceProperty::LoadShmProperty(IFile* shmPropertyFile)
101+
{
102+
auto status = shmPropertyFile->Open(IFile::OpenFlag::READ_WRITE);
103+
if (status.Failure()) { return status; }
104+
constexpr auto retryInterval = std::chrono::milliseconds(100);
105+
constexpr auto maxTryTime = 100;
106+
auto tryTime = 0;
107+
IFile::FileStat stat;
108+
do {
109+
if (tryTime > maxTryTime) {
110+
UC_ERROR("Shm file({}) not ready.", shmPropertyFile->Path());
111+
return Status::Retry();
112+
}
113+
std::this_thread::sleep_for(retryInterval);
114+
status = shmPropertyFile->Stat(stat);
115+
if (status.Failure()) { return status; }
116+
tryTime++;
117+
} while (static_cast<size_t>(stat.st_size) != PropertySize);
118+
status = shmPropertyFile->MMap(this->addr_, PropertySize, true, true, true);
119+
if (status.Failure()) { return status; }
120+
auto property = PropertyPtr(this->addr_);
121+
tryTime = 0;
122+
do {
123+
if (property->magic == Magic) { break; }
124+
if (tryTime > maxTryTime) {
125+
UC_ERROR("Shm file({}) not ready.", shmPropertyFile->Path());
126+
return Status::Retry();
127+
}
128+
std::this_thread::sleep_for(retryInterval);
129+
tryTime++;
130+
} while (true);
131+
return Status::OK();
132+
}
133+
134+
} // namespace UC
Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/**
2+
* MIT License
3+
*
4+
* Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved.
5+
*
6+
* Permission is hereby granted, free of charge, to any person obtaining a copy
7+
* of this software and associated documentation files (the "Software"), to deal
8+
* in the Software without restriction, including without limitation the rights
9+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10+
* copies of the Software, and to permit persons to whom the Software is
11+
* furnished to do so, subject to the following conditions:
12+
*
13+
* The above copyright notice and this permission notice shall be included in all
14+
* copies or substantial portions of the Software.
15+
*
16+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22+
* SOFTWARE.
23+
* */
24+
25+
#ifndef UNIFIEDCACHE_SPACE_PROPERTY_H
26+
#define UNIFIEDCACHE_SPACE_PROPERTY_H
27+
28+
#include "file/ifile.h"
29+
#include "status/status.h"
30+
31+
namespace UC {
32+
33+
class SpaceProperty {
34+
public:
35+
~SpaceProperty();
36+
Status Setup(const std::string& propertyFilePath);
37+
void IncreaseCapacity(const size_t delta);
38+
void DecreaseCapacity(const size_t delta);
39+
size_t GetCapacity() const;
40+
41+
private:
42+
Status InitShmProperty(IFile* shmPropertyFile);
43+
Status LoadShmProperty(IFile* shmPropertyFile);
44+
45+
private:
46+
void* addr_{nullptr};
47+
};
48+
49+
} // namespace UC
50+
51+
#endif

ucm/store/nfsstore/cc/domain/space/space_shard_layout.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,12 @@ class SpaceShardLayout : public SpaceLayout {
3333
Status Setup(const std::vector<std::string>& storageBackends) override;
3434
std::string DataFileParent(const std::string& blockId, bool activated) const override;
3535
std::string DataFilePath(const std::string& blockId, bool activated) const override;
36+
std::string StorageBackend() const { return this->storageBackends_.front(); }
37+
std::string FileRoot() const { return "cluster"; }
38+
std::string PropertyFilePath() const
39+
{
40+
return fmt::format("{}{}.bin", this->StorageBackend(), "uc_property");
41+
}
3642

3743
protected:
3844
virtual std::vector<std::string> RelativeRoots() const;

0 commit comments

Comments
 (0)