Skip to content

Commit 40b61b1

Browse files
author
h0pless
committed
YT-26695: Introduce logical version to Hydra
Introduced a distinction between logical and physical versions of Hydra. First, let's clarify what I mean when I write "physical mutation" and "logical mutation". Physical mutation is an entry in the changelog. Logical mutation is what people usually call "a mutation", a handler to a specific protobuf message, that changes the persistent state of Hydra. Several unrelated logical mutations can be batched in one physical mutation. This may cause problems, since the version of the mutation is taken from the physical mutation. It lead to a few bugs, when people assumed that different logical mutations always have different mutation versions associated with them. To combat that, a clear distinction between the two has been introduced in this PR. commit_hash:1ac7c6a21601191f252c8cfcb8fd8bdbbe479d19
1 parent de7ba2d commit 40b61b1

File tree

4 files changed

+207
-21
lines changed

4 files changed

+207
-21
lines changed

‎yt/yt/client/hydra/version-inl.h‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#ifndef VERSION_INL_H_
2+
#error "Direct inclusion of this file is not allowed, include version.h"
3+
// For the sake of sane code completion.
4+
#include "version.h"
5+
#endif
6+
7+
namespace NYT::NHydra {
8+
9+
////////////////////////////////////////////////////////////////////////////////
10+
11+
template <class TImpl>
12+
[[nodiscard]] TImpl TExtendedVersionBase<TImpl>::Advance(int delta) const
13+
{
14+
YT_VERIFY(delta > 0);
15+
16+
return TImpl(SegmentId, RecordId + delta);
17+
}
18+
19+
template <class TImpl>
20+
[[nodiscard]] TImpl TExtendedVersionBase<TImpl>::Rotate() const
21+
{
22+
return TImpl(SegmentId + 1, 0);
23+
}
24+
25+
////////////////////////////////////////////////////////////////////////////////
26+
27+
} // namespace NYT::NHydra

‎yt/yt/client/hydra/version.cpp‎

Lines changed: 82 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
#include "version.h"
22

3+
#include <yt/yt/core/misc/serialize.h>
4+
35
#include <library/cpp/yt/string/format.h>
46

57
namespace NYT::NHydra {
@@ -58,33 +60,25 @@ TVersion::TVersion(int segmentId, int recordId) noexcept
5860
, RecordId(recordId)
5961
{ }
6062

61-
// std::strong_ordering TVersion::operator <=> (const TVersion& other) const
62-
// {
63-
// if (SegmentId != other.SegmentId) {
64-
// return SegmentId <=> other.SegmentId;
65-
// }
66-
// return RecordId <=> other.RecordId;
67-
// }
68-
6963
TRevision TVersion::ToRevision() const
7064
{
7165
return TRevision((static_cast<ui64>(SegmentId) << 32) | static_cast<ui64>(RecordId));
7266
}
7367

74-
TVersion TVersion::FromRevision(TRevision revision)
68+
void TVersion::Save(TStreamSaveContext& context) const
7569
{
76-
return TVersion(revision.Underlying() >> 32, revision.Underlying() & 0xffffffff);
77-
}
70+
using NYT::Save;
7871

79-
TVersion TVersion::Advance(int delta) const
80-
{
81-
YT_ASSERT(delta >= 0);
82-
return TVersion(SegmentId, RecordId + delta);
72+
Save(context, SegmentId);
73+
Save(context, RecordId);
8374
}
8475

85-
TVersion TVersion::Rotate() const
76+
void TVersion::Load(TStreamLoadContext& context)
8677
{
87-
return TVersion(SegmentId + 1, 0);
78+
using NYT::Load;
79+
80+
Load(context, SegmentId);
81+
Load(context, RecordId);
8882
}
8983

9084
void FormatValue(TStringBuilderBase* builder, TVersion version, TStringBuf /* spec */)
@@ -94,4 +88,75 @@ void FormatValue(TStringBuilderBase* builder, TVersion version, TStringBuf /* sp
9488

9589
////////////////////////////////////////////////////////////////////////////////
9690

91+
TLogicalVersion TLogicalVersion::FromRevision(TRevision revision)
92+
{
93+
return TLogicalVersion(revision.Underlying() >> 32, revision.Underlying() & 0xffffffff);
94+
}
95+
96+
////////////////////////////////////////////////////////////////////////////////
97+
98+
TAutomatonVersion::TAutomatonVersion(
99+
int segmentId,
100+
int physicalRecordId,
101+
int logicalRecordId)
102+
: SegmentId_(segmentId)
103+
, PhysicalRecordId_(physicalRecordId)
104+
, LogicalRecordId_(logicalRecordId)
105+
{
106+
YT_VERIFY(physicalRecordId <= logicalRecordId);
107+
}
108+
109+
TAutomatonVersion::TAutomatonVersion(
110+
TPhysicalVersion physicalVersion,
111+
TLogicalVersion logicalVersion)
112+
: SegmentId_(physicalVersion.SegmentId)
113+
, PhysicalRecordId_(physicalVersion.RecordId)
114+
, LogicalRecordId_(logicalVersion.RecordId)
115+
{
116+
YT_VERIFY(physicalVersion.SegmentId == logicalVersion.SegmentId);
117+
YT_VERIFY(physicalVersion.RecordId <= logicalVersion.RecordId);
118+
}
119+
120+
TPhysicalVersion TAutomatonVersion::GetPhysicalVersion() const
121+
{
122+
return TPhysicalVersion(SegmentId_, PhysicalRecordId_);
123+
}
124+
125+
TLogicalVersion TAutomatonVersion::GetLogicalVersion() const
126+
{
127+
return TLogicalVersion(SegmentId_, LogicalRecordId_);
128+
}
129+
130+
int TAutomatonVersion::GetSegmentId() const
131+
{
132+
return SegmentId_;
133+
}
134+
135+
TRevision TAutomatonVersion::GetLogicalRevision() const
136+
{
137+
return TLogicalVersion(SegmentId_, LogicalRecordId_).ToRevision();
138+
}
139+
140+
TAutomatonVersion TAutomatonVersion::Advance() const
141+
{
142+
return TAutomatonVersion(
143+
std::move(GetPhysicalVersion().Advance()),
144+
std::move(GetLogicalVersion().Advance()));
145+
}
146+
147+
void FormatValue(TStringBuilderBase* builder, TAutomatonVersion version, TStringBuf /* spec */)
148+
{
149+
auto logicalVersion = version.GetLogicalVersion();
150+
auto physicalVersion = version.GetPhysicalVersion();
151+
152+
YT_ASSERT(physicalVersion.SegmentId == logicalVersion.SegmentId);
153+
154+
builder->AppendFormat("%v:%v(%v)",
155+
physicalVersion.SegmentId,
156+
physicalVersion.RecordId,
157+
logicalVersion.RecordId);
158+
}
159+
160+
////////////////////////////////////////////////////////////////////////////////
161+
97162
} // namespace NYT::NHydra

‎yt/yt/client/hydra/version.h‎

Lines changed: 96 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -61,17 +61,110 @@ struct TVersion
6161
std::strong_ordering operator <=> (const TVersion& other) const = default;
6262
bool operator == (const TVersion& other) const = default;
6363

64+
// COMPAT(h0pless): HydraLogicalRecordId. Move this method to TLogicalVersion.
6465
TRevision ToRevision() const;
65-
static TVersion FromRevision(TRevision revision);
6666

67-
TVersion Advance(int delta = 1) const;
68-
TVersion Rotate() const;
67+
void Save(TStreamSaveContext& context) const;
68+
void Load(TStreamLoadContext& context);
6969
};
7070

7171
void FormatValue(TStringBuilderBase* builder, TVersion version, TStringBuf spec);
7272

7373
////////////////////////////////////////////////////////////////////////////////
7474

75+
template <class TImpl>
76+
struct TExtendedVersionBase
77+
: public TVersion
78+
{
79+
using TVersion::TVersion;
80+
81+
[[nodiscard]] TImpl Advance(int delta = 1) const;
82+
[[nodiscard]] TImpl Rotate() const;
83+
};
84+
85+
struct TLogicalVersion
86+
: public TExtendedVersionBase<TLogicalVersion>
87+
{
88+
using TExtendedVersionBase::TExtendedVersionBase;
89+
90+
static TLogicalVersion FromRevision(TRevision revision);
91+
};
92+
93+
struct TPhysicalVersion
94+
: public TExtendedVersionBase<TPhysicalVersion>
95+
{
96+
using TExtendedVersionBase::TExtendedVersionBase;
97+
};
98+
99+
////////////////////////////////////////////////////////////////////////////////
100+
101+
class TAutomatonVersion
102+
{
103+
public:
104+
TAutomatonVersion() = default;
105+
TAutomatonVersion(
106+
int segmentId,
107+
int physicalRecordId,
108+
int logicalRecordId);
109+
110+
TPhysicalVersion GetPhysicalVersion() const;
111+
TLogicalVersion GetLogicalVersion() const;
112+
int GetSegmentId() const;
113+
114+
TRevision GetLogicalRevision() const;
115+
116+
[[nodiscard]] TAutomatonVersion Advance() const;
117+
118+
private:
119+
explicit TAutomatonVersion(
120+
TPhysicalVersion physicalVersion,
121+
TLogicalVersion logicalVersion);
122+
123+
/*
124+
* Sometimes a single physical mutation can consist of several independent
125+
* logical mutations. This means that a single entry in the changelog is
126+
* responsible for a multitude of changes, some of which are not related
127+
* to each other.
128+
*
129+
* This is often the case with Hive mutations.
130+
*
131+
* This results in the following problem: two or more changes happened
132+
* independently on cell C1 and resulted in different versions and
133+
* revisions associated with said changes. But the same logical mutations
134+
* happened in one Hive mutation on cell C2, meaning that now the
135+
* version/revision is the same.
136+
*
137+
* This means that, technically speaking, it's very unsafe to compare
138+
* mutation version and mutation/object revisions in a mutation.
139+
* To remedy this, it's sufficient to introduce a distinction between:
140+
* 1. PhysicalRecordId - changelog entry index; this is the value that
141+
* Hydra uses "under the hood".
142+
* 2. LogicalRecordId - the index that mutation would have had, if every
143+
* mutation consisted of related changes only.
144+
*
145+
* These record IDs are saved in PhysicalVersion and LogicalVersion.
146+
*
147+
* Example:
148+
* Physical Index: +----- 1 -----+--------- 2 --------+------ 3 -----+
149+
* Changelog entry: | CreateNode | ApplyHiveMutations | SetAttribute |
150+
* Logical entry: | CreateNode | StartTx | LockNode | SetAttribute |
151+
* Logical Index: +----- 1 -----+--- 2 ---+---- 3 ---+------ 4 -----+
152+
*
153+
* If you are unsure which version to use - use logical.
154+
*/
155+
int SegmentId_ = 0;
156+
int PhysicalRecordId_ = 0;
157+
int LogicalRecordId_ = 0;
158+
};
159+
160+
void FormatValue(TStringBuilderBase* builder, TAutomatonVersion version, TStringBuf spec);
161+
162+
////////////////////////////////////////////////////////////////////////////////
163+
75164
} // namespace NYT::NHydra
76165

77166
Y_DECLARE_PODTYPE(NYT::NHydra::TVersion);
167+
168+
#define VERSION_INL_H_
169+
#include "version-inl.h"
170+
#undef VERSION_INL_H_

‎yt/yt/client/object_client/helpers-inl.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,11 @@ inline ui32 EntropyFromId(TObjectId id)
3939
return id.Parts32[0];
4040
}
4141

42+
// TODO(h0pless): Replace TVersion with TLogicalVersion.
4243
inline NHydra::TVersion VersionFromId(TObjectId id)
4344
{
4445
YT_ASSERT(!IsSequoiaId(id));
45-
return NHydra::TVersion::FromRevision(RevisionFromId(id));
46+
return NHydra::TLogicalVersion::FromRevision(RevisionFromId(id));
4647
}
4748

4849
inline NTransactionClient::TTimestamp TimestampFromId(TObjectId id)

0 commit comments

Comments
 (0)