Merge branch 'master' into ci-coverage

Author: negokaz

.github/workflows/ci.yml

@@ -7,7 +7,13 @@ on:
     branches: [ master, feature/** ]
 
 env:
-  SBT_OPTS: -Dlerna.enable.discipline
+  # classic timefactor doesn't affect classic default-timeout
+  SBT_OPTS: >-
+    -Dlerna.enable.discipline
+    -Dakka.test.timefactor=3.0
+    -Dakka.actor.testkit.typed.timefactor=3.0
+    -Dakka.test.default-timeout=15s
+    -Dakka.testconductor.barrier-timeout=90s
 
 jobs:
   test:
@@ -51,7 +57,7 @@ jobs:
 
     - name: Run integration tests
       continue-on-error: true # results are reported by action-junit-report
-      run: sh ./scripts/run-multijvm-test.sh 6
+      run: sh ./scripts/run-multijvm-test.sh 1
 
     - name: Publish test report
       uses: mikepenz/action-junit-report@v2

CHANGELOG.md

@@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 [Unreleased]: https://github.com/lerna-stack/akka-entity-replication/compare/v2.0.0...master
 
+### Added
+- Efficient recovery of commit log store, which is on the query side [#112](https://github.com/lerna-stack/akka-entity-replication/issues/112)
+
+  This change will improve the performance of the recovery on the query side.
+  You should migrate settings described at [Migration Guide](docs/migration_guide.md#210-from-200).
+
+- Raft actors start automatically after an initialization of `ClusterReplication` [#118](https://github.com/lerna-stack/akka-entity-replication/issues/118)
+
+  This feature is enabled only by using `typed.ClusterReplication`.
+  It is highly recommended that you switch using the typed API since the classic API was deprecated.
+
 ### Changed
 - Bump up Akka version to 2.6.17 [PR#98](https://github.com/lerna-stack/akka-entity-replication/pull/98)
 
@@ -15,6 +26,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 - TestKit throws "Shard received unexpected message" exception after the entity passivated [PR#100](https://github.com/lerna-stack/akka-entity-replication/pull/100)
+- `ReplicatedEntity` can produce illegal snapshot if compaction and receiving new event occur same time [#111](https://github.com/lerna-stack/akka-entity-replication/issues/111)
+- Starting a follower member later than leader completes a compaction may break ReplicatedLog of the follower [#105](https://github.com/lerna-stack/akka-entity-replication/issues/105)
+- The Raft leader uses the same previous `LogEntryIndex` and `Term` to all batched `AppendEntries` messages [#123](https://github.com/lerna-stack/akka-entity-replication/issues/123)
+- Raft Actors doesn't accept a `RequestVote(lastLogIndex < log.lastLogIndex, lastLogTerm > log.lastLogTerm)` message [#125](https://github.com/lerna-stack/akka-entity-replication/issues/125)
+- A new event is created even though all past events have not been applied [#130](https://github.com/lerna-stack/akka-entity-replication/issues/130)
+- `InstallSnapshot` can miss snapshots to copy [PR#128](https://github.com/lerna-stack/akka-entity-replication/pull/128)
+
+  ⚠️ This change adds a new persistence event. This might don't allow downgrading after upgrading.
 
 ## [v2.0.0] - 2021-07-16
 [v2.0.0]: https://github.com/lerna-stack/akka-entity-replication/compare/v1.0.0...v2.0.0

README.md

@@ -60,6 +60,9 @@ For more information on how to implement an application using this library, plea
 - [Implementation Guide](docs/typed/implementation_guide.md)
 - [Testing Guide](docs/typed/testing_guide.md)
 
+**NOTE**: It is highly recommended that you carefully do cluster operations.
+For more details, please refer to [Operation Guide](docs/operation_guide.md).
+
 ## For Contributors
 
 [CONTRIBUTING](CONTRIBUTING.md) may help us.
@@ -74,6 +77,10 @@ You can see a sample application using this extension in the following project.
 
 You can see all the notable changes in [CHANGELOG](CHANGELOG.md).
 
+## Migration Guide
+
+[Migration Guide](docs/migration_guide.md) describes how to migrate code and settings from previous versions.
+
 ## License
 
 akka-entity-replication is released under the terms of the [Apache License Version 2.0](./LICENSE).

build.sbt

@@ -32,6 +32,8 @@ lazy val lerna = (project in file("."))
     name := "akka-entity-replication",
     fork in Test := true,
     parallelExecution in Test := false,
+    javaOptions in Test ++= sbtJavaOptions,
+    jvmOptions in MultiJvm ++= sbtJavaOptions,
     libraryDependencies ++= Seq(
         "com.typesafe.akka" %% "akka-cluster-typed"    % akkaVersion,
         "com.typesafe.akka" %% "akka-stream"           % akkaVersion,
@@ -51,7 +53,9 @@ lazy val lerna = (project in file("."))
         "com.typesafe.akka" %% "akka-multi-node-testkit"  % akkaVersion % Test,
         // akka-persistence-inmemory が 2.6.x 系に対応していない。
         // TODO 2.6.x 系に対応できる方法に変更する。
-        "com.github.dnvriend" %% "akka-persistence-inmemory" % "2.5.15.2" % Test,
+        "com.github.dnvriend" %% "akka-persistence-inmemory" % "2.5.15.2"  % Test,
+        "com.typesafe.akka"   %% "akka-persistence-testkit"  % akkaVersion % Test,
+        "com.typesafe.akka"   %% "akka-stream-testkit"       % akkaVersion % Test,
       ),
     inConfig(MultiJvm)(
       // multi-jvm ディレクトリをフォーマットするために必要
@@ -82,6 +86,20 @@ lazy val lerna = (project in file("."))
     mimaReportSignatureProblems := true, // check also generic parameters
   )
 
+/**
+  * This is used to pass specific system properties (mostly from CI environment variables)
+  * to the forked process by sbt
+  */
+val sbtJavaOptions: Seq[String] = {
+  // selects properties starting with the following
+  val includes = Set(
+    "akka.",
+  )
+  sys.props.collect {
+    case (k, v) if includes.exists(k.startsWith) => s"-D$k=$v"
+  }.toSeq
+}
+
 addCommandAlias(
   "testCoverage",
   Seq(

docs/implementation_guide.md

@@ -375,6 +375,13 @@ lerna.akka.entityreplication.raft.eventsourced {
       // Absolute path to the journal plugin configuration entry.
       // The journal stores Raft-committed events.
       journal.plugin = ""
+
+      // Absolute path to the snapshot-store plugin configuration entry.
+      // The snapshot-store stores a state (snapshot) built from Raft-committed events.
+      snapshot-store.plugin = ""
+
+      // Snapshot after this number of events.
+      snapshot-every = 1000
     }
 }
 ```
@@ -393,6 +400,7 @@ lerna.akka.entityreplication.raft.persistence {
 // Query side persistence plugin settings
 lerna.akka.entityreplication.raft.eventsourced.persistence {
     journal.plugin  = ""
+    snapshot-store.plugin = ""
 }
 ```
 

docs/migration_guide.md

@@ -0,0 +1,20 @@
+# Migration Guide
+
+## 2.1.0 from 2.0.0
+
+### Configure a snapshot store on the query side
+*akka-entity-replication 2.1.0* introduces an efficient recovery on the query side.
+We've achieved this efficient recovery by using a snapshot feature of Akka persistence.
+This efficient recovery requires you to configure a snapshot store like the following:
+```hocon
+lerna.akka.entityreplication.raft.eventsourced.persistence {
+  snapshot-store.plugin = "Specify your snapshot store plugin ID to use"
+}
+```
+Note that this snapshot store is mandatory.
+You have to configure the snapshot store.
+
+This efficient recovery also introduces new settings named `lerna.akka.entityreplication.raft.eventsourced.persistence.snapshot-every`.
+*akka-entity-replication 2.1.0* saves a snapshot every `snapshot-every` events.
+The default value of `snapshot-every` is 1000.
+You can override this setting according to your requirements.

docs/operation_guide.md

@@ -0,0 +1,68 @@
+# Operation Guide
+
+**Principle: Don't terminate all Raft members at the same time.**
+
+To achieve this principle, you carefully do cluster operations.
+This document supposes that you will use three `multi-raft-roles` (`replica-group-1`, `replica-group-2`, `replica-group-3`).
+
+
+## Start a new cluster
+The important point is to start only one node in at least one role.
+
+A possible full cluster start operation is the following:
+* Start a node (or multiple nodes) with role `replica-group-1`.
+* Start a node (or multiple nodes) with role `replica-group-2`.
+* Start exactly one node with `replica-group-3`.
+
+You can do the above operations simultaneously.
+You have to wait for all Raft members to be running
+before adding more nodes with `replica-group-3`.
+The waiting time depends on your settings, such as the following:
+* `lerna.akka.entityreplication.raft.election-timeout`
+* `lerna.akka.entityreplication.raft.heartbeat-interval`
+* `lerna.akka.entityreplication.raft.number-of-shards`
+* `lerna.akka.entityreplication.raft.raft-actor-auto-start`
+
+
+## Add nodes
+Don't add nodes with different roles at the same time.
+
+Any of the following operations is possible:
+* Add a node (or multiple nodes) with `replica-group-1`.
+* Add a node (or multiple nodes) with `replica-group-2`.
+* Add a node (or multiple nodes) with `replica-group-3`.
+
+You **should not** do the above operations simultaneously.
+You have to wait for all Raft members to be running
+before adding more nodes or removing nodes.
+The waiting time depends on your settings, such as the following:
+* `lerna.akka.entityreplication.raft.election-timeout`
+* `lerna.akka.entityreplication.raft.heartbeat-interval`
+
+Any of the following operations is possible but **not recommended**.
+It is because you cannot ensure that any of nodes with untouched roles won't crash.
+* Add nodes with `replica-group-1` and nodes with `replica-group-2`.
+* Add nodes with `replica-group-2` and nodes with `replica-group-3`.
+* Add nodes with `replica-group-3` and nodes with `replica-group-1`.
+
+
+## Remove nodes
+Don't remove nodes with different roles at the same time.
+
+Any of the following operations is possible:
+* Remove a node (or multiple nodes) with `replica-group-1`.
+* Remove a node (or multiple nodes) with `replica-group-2`.
+* Remove a node (or multiple nodes) with `replica-group-3`.
+
+You **should not** do the above operations simultaneously.
+You have to wait for all Raft members to be running
+before adding nodes or removing more nodes.
+Waiting time depends on your settings, such as the following:
+* `lerna.akka.entityreplication.raft.election-timeout`
+* `lerna.akka.entityreplication.raft.heartbeat-interval`
+
+Any of the following operations is possible but **not recommended**.
+It is because you cannot ensure that any of nodes with untouched roles won't crash.
+* Remove nodes with `replica-group-1` and nodes with `replica-group-2`.
+* Remove nodes with `replica-group-2` and nodes with `replica-group-3`.
+* Remove nodes with `replica-group-3` and nodes with `replica-group-1`.

docs/typed/implementation_guide.md

@@ -327,9 +327,10 @@ lerna.akka.entityreplication.raft.persistence.journal.plugin = ""
 lerna.akka.entityreplication.raft.persistence.snapshot-store.plugin = ""
 lerna.akka.entityreplication.raft.persistence.query.plugin = ""
 lerna.akka.entityreplication.raft.eventsourced.persistence.journal.plugin = ""
+lerna.akka.entityreplication.raft.eventsourced.persistence.snapshot-store.plugin = ""
 ```
 
-You can override these settings by `withRaftJournalPluginId`, `withRaftSnapshotPluginId`, `withRaftQueryPluginId` and `withEventSourcedJournalPluginId`
+You can override these settings by `withRaftJournalPluginId`, `withRaftSnapshotPluginId`, `withRaftQueryPluginId`, `withEventSourcedJournalPluginId`, and `withEventSourcedSnapshotStorePluginId`
 of `ClusterReplicationSettings`.
 
 ```scala
@@ -346,6 +347,7 @@ val settings =
     .withRaftSnapshotPluginId("my.special.raft.snapshot-store")
     .withRaftQueryPluginId("my.special.raft.query")
     .withEventSourcedJournalPluginId("my.special.eventsourced.journal")
+    .withEventSourcedSnapshotStorePluginId("my.special.eventsourced.snapshot-store")
 
 val entity = 
   ReplicatedEntity(BankAccountBehavior.TypeKey)(entityContext => BankAccountBehavior(entityContext))
@@ -596,6 +598,7 @@ lerna.akka.entityreplication.raft.persistence {
 // Query side persistence plugin settings
 lerna.akka.entityreplication.raft.eventsourced.persistence {
     journal.plugin  = ""
+    snapshot-store.plugin = ""
 }
 ```
 

src/main/mima-filters/2.0.0.backwards.excludes/pr-114-efficient-recovery-of-commit-log-store.excludes

@@ -0,0 +1,3 @@
+# ClusterReplicationSettings should not be extended by users
+ProblemFilters.exclude[ReversedMissingMethodProblem]("lerna.akka.entityreplication.ClusterReplicationSettings.withEventSourcedSnapshotStorePluginId")
+ProblemFilters.exclude[ReversedMissingMethodProblem]("lerna.akka.entityreplication.typed.ClusterReplicationSettings.withEventSourcedSnapshotStorePluginId")

src/main/mima-filters/2.0.0.backwards.excludes/pr-119-raft-actor-auto-start.excludes

@@ -0,0 +1,6 @@
+# It is safe to exclude the following since ClusterReplicationGuardian is package-private.
+ProblemFilters.exclude[DirectMissingMethodProblem]("lerna.akka.entityreplication.ClusterReplicationGuardian#Start.copy")
+ProblemFilters.exclude[DirectMissingMethodProblem]("lerna.akka.entityreplication.ClusterReplicationGuardian#Start.this")
+ProblemFilters.exclude[MissingTypesProblem]("lerna.akka.entityreplication.ClusterReplicationGuardian$Start$")
+ProblemFilters.exclude[DirectMissingMethodProblem]("lerna.akka.entityreplication.ClusterReplicationGuardian#Start.apply")
+ProblemFilters.exclude[IncompatibleSignatureProblem]("lerna.akka.entityreplication.ClusterReplicationGuardian#Start.unapply")